From c9c3512cc7516bf4fbc828d9836d9bb5b376f4c4 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 16 Dec 2023 11:53:59 +0000 Subject: [PATCH 01/80] Move post-processing modules into separate packages Avoids needing to import unnecessary dependencies (Plots.jl when using Makie.jl, or Makie.jl when using Plots.jl, or either when only running simulations), which should reduce precompilation time, and the time to build system-images. Also removes a few unused dependencies. --- Project.toml | 11 - makie_post_processing/Project.toml | 16 ++ .../src}/makie_post_processing.jl | 49 ++-- makie_post_processing/src/shared_utils.jl | 203 ++++++++++++++ plots_post_processing/Project.toml | 20 ++ .../post_processing_input.jl | 1 + .../src}/plot_MMS_sequence.jl | 32 +-- .../src}/plot_sequence.jl | 20 +- .../src/plots_post_processing.jl | 251 +++--------------- .../src}/post_processing_input.jl | 4 +- plots_post_processing/src/shared_utils.jl | 1 + src/fokker_planck_test.jl | 38 +-- src/moment_kinetics.jl | 5 - 13 files changed, 356 insertions(+), 295 deletions(-) create mode 100644 makie_post_processing/Project.toml rename {src => makie_post_processing/src}/makie_post_processing.jl (99%) create mode 100644 makie_post_processing/src/shared_utils.jl create mode 100644 plots_post_processing/Project.toml create mode 120000 plots_post_processing/post_processing_input.jl rename {src => plots_post_processing/src}/plot_MMS_sequence.jl (93%) rename {src => plots_post_processing/src}/plot_sequence.jl (85%) rename src/post_processing.jl => plots_post_processing/src/plots_post_processing.jl (95%) rename {src => plots_post_processing/src}/post_processing_input.jl (99%) create mode 120000 plots_post_processing/src/shared_utils.jl diff --git a/Project.toml b/Project.toml index 97abfed83..49535c119 100644 --- a/Project.toml +++ b/Project.toml @@ -5,39 +5,28 @@ version = "0.1.0" [deps] ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" -CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" FastGaussQuadrature = "442a2c76-b920-505d-bb47-c5924d526838" -GR_jll = "d2c73de3-f751-5644-a686-071e5b155ba9" Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" HDF5_jll = "0234f1f7-429e-5d53-9886-15a909be8d59" -IJulia = "7073ff75-c697-5162-941a-fcdaad2a7d2a" IfElse = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173" Interpolations = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59" -LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" LegendrePolynomials = "3db4a2ba-fc88-11e8-3e01-49c72059a882" LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" -LsqFit = "2fda8390-95c7-5789-9bda-21331edee243" MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" Measures = "442fdcdd-2543-5da2-b0f3-8c86c306513e" NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" -NaNMath = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" -NaturalSort = "c020b1a1-e9b0-503a-9c33-f039bfc54a85" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" PackageCompiler = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Preferences = "21216c6a-2e73-6563-6e65-726566657250" Primes = "27ebfcd6-29c5-5fa9-bf4b-fb8fc14df3ae" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" -PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" -PyPlot = "d330b81b-6aea-500a-939a-2ce795aea3ee" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" Roots = "f2b01f46-fcfa-551c-844a-d8ac1e96c665" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" diff --git a/makie_post_processing/Project.toml b/makie_post_processing/Project.toml new file mode 100644 index 000000000..4416f6ed1 --- /dev/null +++ b/makie_post_processing/Project.toml @@ -0,0 +1,16 @@ +name = "makie_post_processing" +uuid = "4dd1b173-c370-4c56-9cc2-d797e41ae9f0" +authors = ["John Omotani "] +version = "0.1.0" + +[deps] +CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" +Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" +LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +LsqFit = "2fda8390-95c7-5789-9bda-21331edee243" +MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" +NaNMath = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +moment_kinetics = "b5ff72cc-06fc-4161-ad14-dba1c22ed34e" diff --git a/src/makie_post_processing.jl b/makie_post_processing/src/makie_post_processing.jl similarity index 99% rename from src/makie_post_processing.jl rename to makie_post_processing/src/makie_post_processing.jl index 6cacb3a30..5545c4373 100644 --- a/src/makie_post_processing.jl +++ b/makie_post_processing/src/makie_post_processing.jl @@ -17,27 +17,34 @@ export animate_f_unnorm_vs_vpa, animate_f_unnorm_vs_vpa_z, get_1d_ax, get_2d_ax, plot_f_unnorm_vs_vpa_z, positive_or_nan, postproc_load_variable, positive_or_nan, put_legend_above, put_legend_below, put_legend_left, put_legend_right -using ..analysis: analyze_fields_data, check_Chodura_condition, get_r_perturbation, - get_Fourier_modes_2D, get_Fourier_modes_1D, steady_state_residuals, - get_unnormalised_f_dzdt_1d, get_unnormalised_f_coords_2d, - get_unnormalised_f_1d, vpagrid_to_dzdt_2d, get_unnormalised_f_2d -using ..array_allocation: allocate_float -using ..coordinates: define_coordinate -using ..input_structs: grid_input, advection_input, set_defaults_and_check_top_level!, - set_defaults_and_check_section!, Dict_to_NamedTuple -using ..krook_collisions: get_collision_frequency -using ..looping: all_dimensions, ion_dimensions, neutral_dimensions -using ..manufactured_solns: manufactured_solutions, manufactured_electric_fields -using ..moment_kinetics_input: mk_input -using ..load_data: open_readonly_output_file, get_group, load_block_data, - load_coordinate_data, load_distributed_charged_pdf_slice, - load_distributed_neutral_pdf_slice, load_input, load_mk_options, - load_species_data, load_time_data -using ..initial_conditions: vpagrid_to_dzdt -using ..post_processing: calculate_and_write_frequencies, construct_global_zr_coords, - get_geometry_and_composition, read_distributed_zr_data! -using ..type_definitions: mk_float, mk_int -using ..velocity_moments: integrate_over_vspace, integrate_over_neutral_vspace +include("shared_utils.jl") + +using moment_kinetics.analysis: analyze_fields_data, check_Chodura_condition, + get_r_perturbation, get_Fourier_modes_2D, + get_Fourier_modes_1D, steady_state_residuals, + get_unnormalised_f_dzdt_1d, get_unnormalised_f_coords_2d, + get_unnormalised_f_1d, vpagrid_to_dzdt_2d, + get_unnormalised_f_2d +using moment_kinetics.array_allocation: allocate_float +using moment_kinetics.coordinates: define_coordinate +using moment_kinetics.input_structs: grid_input, advection_input, + set_defaults_and_check_top_level!, + set_defaults_and_check_section!, Dict_to_NamedTuple +using moment_kinetics.krook_collisions: get_collision_frequency +using moment_kinetics.looping: all_dimensions, ion_dimensions, neutral_dimensions +using moment_kinetics.manufactured_solns: manufactured_solutions, + manufactured_electric_fields +using moment_kinetics.moment_kinetics_input: mk_input +using moment_kinetics.load_data: open_readonly_output_file, get_group, load_block_data, + load_coordinate_data, load_distributed_charged_pdf_slice, + load_distributed_neutral_pdf_slice, load_input, + load_mk_options, load_species_data, load_time_data +using moment_kinetics.initial_conditions: vpagrid_to_dzdt +using .shared_utils: calculate_and_write_frequencies, construct_global_zr_coords, + get_geometry_and_composition, read_distributed_zr_data! +using moment_kinetics.type_definitions: mk_float, mk_int +using moment_kinetics.velocity_moments: integrate_over_vspace, + integrate_over_neutral_vspace using Combinatorics using Glob diff --git a/makie_post_processing/src/shared_utils.jl b/makie_post_processing/src/shared_utils.jl new file mode 100644 index 000000000..d4b33e465 --- /dev/null +++ b/makie_post_processing/src/shared_utils.jl @@ -0,0 +1,203 @@ +module shared_utils + +export calculate_and_write_frequencies, construct_global_zr_coords, + get_geometry_and_composition, read_distributed_zr_data! + +using moment_kinetics.coordinates: define_coordinate +using moment_kinetics.input_structs: grid_input +using moment_kinetics.type_definitions: mk_float, mk_int + +using MPI + +""" +""" +function calculate_and_write_frequencies(run_name, ntime, time, z, itime_min, itime_max, + iz0, delta_phi, pp) + if pp.calculate_frequencies + println("Calculating the frequency and damping/growth rate...") + # shifted_time = t - t0 + shifted_time = allocate_float(ntime) + @. shifted_time = time - time[itime_min] + # assume phi(z0,t) = A*exp(growth_rate*t)*cos(ω*t + φ) + # and fit phi(z0,t)/phi(z0,t0), which eliminates the constant A pre-factor + @views phi_fit = fit_delta_phi_mode(shifted_time[itime_min:itime_max], z, + delta_phi[:, itime_min:itime_max]) + frequency = phi_fit.frequency + growth_rate = phi_fit.growth_rate + + # write info related to fit to file + io = open_ascii_output_file(run_name, "frequency_fit.txt") + println(io, "#growth_rate: ", phi_fit.growth_rate, + " frequency: ", phi_fit.frequency, + " fit_errors: ", phi_fit.amplitude_fit_error, " ", + phi_fit.offset_fit_error, " ", phi_fit.cosine_fit_error) + println(io) + + # Calculate the fitted phi as a function of time at index iz0 + L = z[end] - z[begin] + fitted_delta_phi = + @. (phi_fit.amplitude0 * cos(2.0 * π * (z[iz0] + phi_fit.offset0) / L) + * exp(phi_fit.growth_rate * shifted_time) + * cos(phi_fit.frequency * shifted_time + phi_fit.phase)) + for i ∈ 1:ntime + println(io, "time: ", time[i], " delta_phi: ", delta_phi[iz0,i], + " fitted_delta_phi: ", fitted_delta_phi[i]) + end + close(io) + else + frequency = 0.0 + growth_rate = 0.0 + phase = 0.0 + shifted_time = allocate_float(ntime) + @. shifted_time = time - time[itime_min] + fitted_delta_phi = zeros(ntime) + + end + return frequency, growth_rate, shifted_time, fitted_delta_phi +end + +""" +""" +function construct_global_zr_coords(r_local, z_local) + + function make_global_input(coord_local) + return grid_input(coord_local.name, coord_local.ngrid, + coord_local.nelement_global, coord_local.nelement_global, 1, 0, coord_local.L, + coord_local.discretization, coord_local.fd_option, coord_local.cheb_option, coord_local.bc, + coord_local.advection, MPI.COMM_NULL, coord_local.element_spacing_option) + end + + r_global, r_global_spectral = define_coordinate(make_global_input(r_local)) + z_global, z_global_spectral = define_coordinate(make_global_input(z_local)) + + return r_global, r_global_spectral, z_global, z_global_spectral +end + +""" +""" +function get_geometry_and_composition(scan_input,n_ion_species,n_neutral_species) + # set geometry_input + # MRH need to get this in way that does not duplicate code + # MRH from moment_kinetics_input.jl + Bzed = get(scan_input, "Bzed", 1.0) + Bmag = get(scan_input, "Bmag", 1.0) + bzed = Bzed/Bmag + bzeta = sqrt(1.0 - bzed^2.0) + Bzeta = Bmag*bzeta + rhostar = get(scan_input, "rhostar", 0.0) + geometry = geometry_input(Bzed,Bmag,bzed,bzeta,Bzeta,rhostar) + + # set composition input + # MRH need to get this in way that does not duplicate code + # MRH from moment_kinetics_input.jl + electron_physics = get(scan_input, "electron_physics", boltzmann_electron_response) + + if electron_physics ∈ (boltzmann_electron_response, boltzmann_electron_response_with_simple_sheath) + n_species = n_ion_species + n_neutral_species + else + n_species = n_ion_species + n_neutral_species + 1 + end + T_e = get(scan_input, "T_e", 1.0) + # set wall temperature T_wall = Tw/Te + T_wall = get(scan_input, "T_wall", 1.0) + # set initial neutral temperature Tn/Tₑ = 1 + # set initial nᵢ/Nₑ = 1.0 + # set phi_wall at z = 0 + phi_wall = get(scan_input, "phi_wall", 0.0) + # if false use true Knudsen cosine for neutral wall bc + use_test_neutral_wall_pdf = get(scan_input, "use_test_neutral_wall_pdf", false) + # constant to be used to test nonzero Er in wall boundary condition + Er_constant = get(scan_input, "Er_constant", 0.0) + recycling_fraction = get(scan_input, "recycling_fraction", 1.0) + # constant to be used to control Ez divergences + epsilon_offset = get(scan_input, "epsilon_offset", 0.001) + # bool to control if dfni is a function of vpa or vpabar in MMS test + use_vpabar_in_mms_dfni = get(scan_input, "use_vpabar_in_mms_dfni", true) + if use_vpabar_in_mms_dfni + alpha_switch = 1.0 + else + alpha_switch = 0.0 + end + # ratio of the neutral particle mass to the ion particle mass + mn_over_mi = 1.0 + # ratio of the electron particle mass to the ion particle mass + me_over_mi = 1.0/1836.0 + composition = species_composition(n_species, n_ion_species, n_neutral_species, + electron_physics, use_test_neutral_wall_pdf, T_e, T_wall, phi_wall, Er_constant, + mn_over_mi, me_over_mi, recycling_fraction, allocate_float(n_species)) + return geometry, composition + +end + +""" +Read data which is a function of (z,r,t) or (z,r,species,t) + +run_names is a tuple. If it has more than one entry, this means that there are multiple +restarts (which are sequential in time), so concatenate the data from each entry together. +""" +function read_distributed_zr_data!(var::Array{mk_float,N}, var_name::String, + run_names::Tuple, file_key::String, nblocks::Tuple, + nz_local::mk_int,nr_local::mk_int,iskip::mk_int) where N + # dimension of var is [z,r,species,t] + + local_tind_start = 1 + local_tind_end = -1 + global_tind_start = 1 + global_tind_end = -1 + for (run_name, nb) in zip(run_names, nblocks) + for iblock in 0:nb-1 + fid = open_readonly_output_file(run_name,file_key,iblock=iblock,printout=false) + group = get_group(fid, "dynamic_data") + var_local = load_variable(group, var_name) + + ntime_local = size(var_local, N) + + # offset is the amount we have to skip at the beginning of this restart to + # line up properly with having outputs every iskip since the beginning of the + # first restart. + # Note: use rem(x,y,RoundDown) here because this gives a result that's + # definitely between 0 and y, whereas rem(x,y) or mod(x,y) give negative + # results for negative x. + offset = rem(1 - (local_tind_start-1), iskip, RoundDown) + if offset == 0 + # Actually want offset in the range [1,iskip], so correct if rem() + # returned 0 + offset = iskip + end + if local_tind_start > 1 + # The run being loaded is a restart (as local_tind_start=1 for the first + # run), so skip the first point, as this is a duplicate of the last point + # of the previous restart + offset += 1 + end + + local_tind_end = local_tind_start + ntime_local - 1 + global_tind_end = global_tind_start + length(offset:iskip:ntime_local) - 1 + + z_irank, z_nrank, r_irank, r_nrank = load_rank_data(fid) + + # min index set to avoid double assignment of repeated points + # 1 if irank = 0, 2 otherwise + imin_r = min(1,r_irank) + 1 + imin_z = min(1,z_irank) + 1 + for ir_local in imin_r:nr_local + for iz_local in imin_z:nz_local + ir_global = iglobal_func(ir_local,r_irank,nr_local) + iz_global = iglobal_func(iz_local,z_irank,nz_local) + if N == 4 + var[iz_global,ir_global,:,global_tind_start:global_tind_end] .= var_local[iz_local,ir_local,:,offset:iskip:end] + elseif N == 3 + var[iz_global,ir_global,global_tind_start:global_tind_end] .= var_local[iz_local,ir_local,offset:iskip:end] + else + error("Unsupported number of dimensions: $N") + end + end + end + close(fid) + end + local_tind_start = local_tind_end + 1 + global_tind_start = global_tind_end + 1 + end +end + +end # shared_utils.jl diff --git a/plots_post_processing/Project.toml b/plots_post_processing/Project.toml new file mode 100644 index 000000000..5d0a85d0f --- /dev/null +++ b/plots_post_processing/Project.toml @@ -0,0 +1,20 @@ +name = "plots_post_processing" +uuid = "c3e9b6e6-652c-46aa-9ac0-1bc582aad122" +authors = ["John Omotani "] +version = "0.1.0" + +[deps] +Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" +IJulia = "7073ff75-c697-5162-941a-fcdaad2a7d2a" +LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +LsqFit = "2fda8390-95c7-5789-9bda-21331edee243" +MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" +Measures = "442fdcdd-2543-5da2-b0f3-8c86c306513e" +NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" +Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" +PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" +PyPlot = "d330b81b-6aea-500a-939a-2ce795aea3ee" +SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +moment_kinetics = "b5ff72cc-06fc-4161-ad14-dba1c22ed34e" diff --git a/plots_post_processing/post_processing_input.jl b/plots_post_processing/post_processing_input.jl new file mode 120000 index 000000000..77acb92c8 --- /dev/null +++ b/plots_post_processing/post_processing_input.jl @@ -0,0 +1 @@ +src/post_processing_input.jl \ No newline at end of file diff --git a/src/plot_MMS_sequence.jl b/plots_post_processing/src/plot_MMS_sequence.jl similarity index 93% rename from src/plot_MMS_sequence.jl rename to plots_post_processing/src/plot_MMS_sequence.jl index d11a53c10..21ba269b6 100644 --- a/src/plot_MMS_sequence.jl +++ b/plots_post_processing/src/plot_MMS_sequence.jl @@ -14,22 +14,22 @@ using SpecialFunctions: erfi using LaTeXStrings # modules using ..post_processing_input: pp -using ..post_processing: compare_charged_pdf_symbolic_test, compare_fields_symbolic_test -using ..post_processing: compare_moments_symbolic_test, compare_neutral_pdf_symbolic_test -using ..post_processing: read_distributed_zr_data!, construct_global_zr_coords -using ..post_processing: allocate_global_zr_neutral_moments, allocate_global_zr_charged_moments -using ..post_processing: allocate_global_zr_fields, get_geometry_and_composition -using ..post_processing: get_coords_nelement, get_coords_ngrid -using ..array_allocation: allocate_float -using ..type_definitions: mk_float, mk_int -using ..load_data: open_readonly_output_file -using ..load_data: load_fields_data, load_pdf_data -using ..load_data: load_charged_particle_moments_data, load_neutral_particle_moments_data -using ..load_data: load_neutral_pdf_data, load_time_data, load_species_data -using ..load_data: load_block_data, load_coordinate_data, load_input -using ..velocity_moments: integrate_over_vspace -using ..manufactured_solns: manufactured_solutions, manufactured_electric_fields -using ..moment_kinetics_input: mk_input, read_input_file +using ..plots_post_processing: compare_charged_pdf_symbolic_test, compare_fields_symbolic_test +using ..plots_post_processing: compare_moments_symbolic_test, compare_neutral_pdf_symbolic_test +using ..plots_post_processing: read_distributed_zr_data!, construct_global_zr_coords +using ..plots_post_processing: allocate_global_zr_neutral_moments, allocate_global_zr_charged_moments +using ..plots_post_processing: allocate_global_zr_fields, get_geometry_and_composition +using ..plots_post_processing: get_coords_nelement, get_coords_ngrid +using moment_kinetics.array_allocation: allocate_float +using moment_kinetics.type_definitions: mk_float, mk_int +using moment_kinetics.load_data: open_readonly_output_file +using moment_kinetics.load_data: load_fields_data, load_pdf_data +using moment_kinetics.load_data: load_charged_particle_moments_data, load_neutral_particle_moments_data +using moment_kinetics.load_data: load_neutral_pdf_data, load_time_data, load_species_data +using moment_kinetics.load_data: load_block_data, load_coordinate_data, load_input +using moment_kinetics.velocity_moments: integrate_over_vspace +using moment_kinetics.manufactured_solns: manufactured_solutions, manufactured_electric_fields +using moment_kinetics.moment_kinetics_input: mk_input, read_input_file import Base: get diff --git a/src/plot_sequence.jl b/plots_post_processing/src/plot_sequence.jl similarity index 85% rename from src/plot_sequence.jl rename to plots_post_processing/src/plot_sequence.jl index 0c370c8a6..8fbf8180b 100644 --- a/src/plot_sequence.jl +++ b/plots_post_processing/src/plot_sequence.jl @@ -11,16 +11,16 @@ using Statistics: mean using SpecialFunctions: erfi using LaTeXStrings # modules -using ..post_processing: read_distributed_zr_data!, construct_global_zr_coords -using ..post_processing: allocate_global_zr_neutral_moments, allocate_global_zr_charged_moments -using ..post_processing: allocate_global_zr_fields#, get_coords_nelement -using ..array_allocation: allocate_float -using ..type_definitions: mk_float, mk_int -using ..load_data: open_readonly_output_file -using ..load_data: load_fields_data -using ..load_data: load_time_data, load_species_data -using ..load_data: load_block_data, load_coordinate_data -using ..moment_kinetics_input: mk_input, read_input_file +using ..plots_post_processing: read_distributed_zr_data!, construct_global_zr_coords +using ..plots_post_processing: allocate_global_zr_neutral_moments, allocate_global_zr_charged_moments +using ..plots_post_processing: allocate_global_zr_fields#, get_coords_nelement +using moment_kinetics.array_allocation: allocate_float +using moment_kinetics.type_definitions: mk_float, mk_int +using moment_kinetics.load_data: open_readonly_output_file +using moment_kinetics.load_data: load_fields_data +using moment_kinetics.load_data: load_time_data, load_species_data +using moment_kinetics.load_data: load_block_data, load_coordinate_data +using moment_kinetics.moment_kinetics_input: mk_input, read_input_file function plot_sequence_fields_data(path_list) diff --git a/src/post_processing.jl b/plots_post_processing/src/plots_post_processing.jl similarity index 95% rename from src/post_processing.jl rename to plots_post_processing/src/plots_post_processing.jl index 48c1b10ec..e9ccbe8c7 100644 --- a/src/post_processing.jl +++ b/plots_post_processing/src/plots_post_processing.jl @@ -1,6 +1,6 @@ """ """ -module post_processing +module plots_post_processing export analyze_and_plot_data export compare_charged_pdf_symbolic_test @@ -14,6 +14,9 @@ export allocate_global_zr_fields export get_coords_nelement export get_coords_ngrid +include("post_processing_input.jl") +include("shared_utils.jl") + # Next three lines only used for workaround needed by plot_unnormalised() using PyCall import PyPlot @@ -30,30 +33,38 @@ using LaTeXStrings using Measures using MPI # modules -using ..post_processing_input: pp -using ..communication -using ..quadrature: composite_simpson_weights -using ..array_allocation: allocate_float -using ..coordinates: define_coordinate -using ..file_io: open_ascii_output_file -using ..type_definitions: mk_float, mk_int -using ..load_data: open_readonly_output_file, get_group, load_input, load_time_data -using ..load_data: get_nranks -using ..load_data: load_fields_data, load_pdf_data -using ..load_data: load_charged_particle_moments_data, load_neutral_particle_moments_data -using ..load_data: load_distributed_charged_pdf_slice, load_distributed_neutral_pdf_slice, iglobal_func -using ..load_data: load_neutral_pdf_data -using ..load_data: load_variable -using ..load_data: load_coordinate_data, load_block_data, load_rank_data, - load_species_data, load_mk_options -using ..analysis: analyze_fields_data, analyze_moments_data, analyze_pdf_data, - check_Chodura_condition, analyze_2D_instability, - get_unnormalised_f_dzdt_1d, get_unnormalised_f_coords_2d -using ..velocity_moments: integrate_over_vspace -using ..manufactured_solns: manufactured_solutions, manufactured_electric_fields -using ..moment_kinetics_input: mk_input, get -using ..input_structs: geometry_input, grid_input, species_composition -using ..input_structs: electron_physics_type, boltzmann_electron_response, boltzmann_electron_response_with_simple_sheath +using moment_kinetics.communication +using moment_kinetics.quadrature: composite_simpson_weights +using moment_kinetics.array_allocation: allocate_float +using moment_kinetics.coordinates: define_coordinate +using moment_kinetics.file_io: open_ascii_output_file +using moment_kinetics.type_definitions: mk_float, mk_int +using moment_kinetics.load_data: open_readonly_output_file, get_group, load_input, + load_time_data +using moment_kinetics.load_data: get_nranks +using moment_kinetics.load_data: load_fields_data, load_pdf_data +using moment_kinetics.load_data: load_charged_particle_moments_data, + load_neutral_particle_moments_data +using moment_kinetics.load_data: load_distributed_charged_pdf_slice, + load_distributed_neutral_pdf_slice, iglobal_func +using moment_kinetics.load_data: load_neutral_pdf_data +using moment_kinetics.load_data: load_variable +using moment_kinetics.load_data: load_coordinate_data, load_block_data, load_rank_data, + load_species_data, load_mk_options +using moment_kinetics.analysis: analyze_fields_data, analyze_moments_data, + analyze_pdf_data, check_Chodura_condition, + analyze_2D_instability, get_unnormalised_f_dzdt_1d, + get_unnormalised_f_coords_2d +using moment_kinetics.velocity_moments: integrate_over_vspace +using moment_kinetics.manufactured_solns: manufactured_solutions, + manufactured_electric_fields +using moment_kinetics.moment_kinetics_input: mk_input, get +using moment_kinetics.input_structs: geometry_input, grid_input, species_composition +using moment_kinetics.input_structs: electron_physics_type, boltzmann_electron_response, + boltzmann_electron_response_with_simple_sheath +using .post_processing_input: pp +using .shared_utils: calculate_and_write_frequencies, construct_global_zr_coords, + get_geometry_and_composition, read_distributed_zr_data! using TOML import Base: get @@ -123,77 +134,6 @@ function trygif(anim, outfile; kwargs...) return nothing end -""" -Read data which is a function of (z,r,t) or (z,r,species,t) - -run_names is a tuple. If it has more than one entry, this means that there are multiple -restarts (which are sequential in time), so concatenate the data from each entry together. -""" -function read_distributed_zr_data!(var::Array{mk_float,N}, var_name::String, - run_names::Tuple, file_key::String, nblocks::Tuple, - nz_local::mk_int,nr_local::mk_int,iskip::mk_int) where N - # dimension of var is [z,r,species,t] - - local_tind_start = 1 - local_tind_end = -1 - global_tind_start = 1 - global_tind_end = -1 - for (run_name, nb) in zip(run_names, nblocks) - for iblock in 0:nb-1 - fid = open_readonly_output_file(run_name,file_key,iblock=iblock,printout=false) - group = get_group(fid, "dynamic_data") - var_local = load_variable(group, var_name) - - ntime_local = size(var_local, N) - - # offset is the amount we have to skip at the beginning of this restart to - # line up properly with having outputs every iskip since the beginning of the - # first restart. - # Note: use rem(x,y,RoundDown) here because this gives a result that's - # definitely between 0 and y, whereas rem(x,y) or mod(x,y) give negative - # results for negative x. - offset = rem(1 - (local_tind_start-1), iskip, RoundDown) - if offset == 0 - # Actually want offset in the range [1,iskip], so correct if rem() - # returned 0 - offset = iskip - end - if local_tind_start > 1 - # The run being loaded is a restart (as local_tind_start=1 for the first - # run), so skip the first point, as this is a duplicate of the last point - # of the previous restart - offset += 1 - end - - local_tind_end = local_tind_start + ntime_local - 1 - global_tind_end = global_tind_start + length(offset:iskip:ntime_local) - 1 - - z_irank, z_nrank, r_irank, r_nrank = load_rank_data(fid) - - # min index set to avoid double assignment of repeated points - # 1 if irank = 0, 2 otherwise - imin_r = min(1,r_irank) + 1 - imin_z = min(1,z_irank) + 1 - for ir_local in imin_r:nr_local - for iz_local in imin_z:nz_local - ir_global = iglobal_func(ir_local,r_irank,nr_local) - iz_global = iglobal_func(iz_local,z_irank,nz_local) - if N == 4 - var[iz_global,ir_global,:,global_tind_start:global_tind_end] .= var_local[iz_local,ir_local,:,offset:iskip:end] - elseif N == 3 - var[iz_global,ir_global,global_tind_start:global_tind_end] .= var_local[iz_local,ir_local,offset:iskip:end] - else - error("Unsupported number of dimensions: $N") - end - end - end - close(fid) - end - local_tind_start = local_tind_end + 1 - global_tind_start = global_tind_end + 1 - end -end - """ Read data which is a function of (r,t) or (r,species,t) and associated to one of the wall boundaries @@ -265,21 +205,6 @@ function read_distributed_zwallr_data!(var::Array{mk_float,N}, var_name::String, end end -function construct_global_zr_coords(r_local, z_local) - - function make_global_input(coord_local) - return grid_input(coord_local.name, coord_local.ngrid, - coord_local.nelement_global, coord_local.nelement_global, 1, 0, coord_local.L, - coord_local.discretization, coord_local.fd_option, coord_local.cheb_option, coord_local.bc, - coord_local.advection, MPI.COMM_NULL, coord_local.element_spacing_option) - end - - r_global, r_global_spectral = define_coordinate(make_global_input(r_local)) - z_global, z_global_spectral = define_coordinate(make_global_input(z_local)) - - return r_global, r_global_spectral, z_global, z_global_spectral -end - """ functions to allocate arrays that are used at run-time to postprocess data that is stored in the netcdf files @@ -351,60 +276,6 @@ function get_coords_ngrid(scan_input) return z_ngrid, r_ngrid, vpa_ngrid, vperp_ngrid, vz_ngrid, vr_ngrid, vzeta_ngrid end -function get_geometry_and_composition(scan_input,n_ion_species,n_neutral_species) - # set geometry_input - # MRH need to get this in way that does not duplicate code - # MRH from moment_kinetics_input.jl - Bzed = get(scan_input, "Bzed", 1.0) - Bmag = get(scan_input, "Bmag", 1.0) - bzed = Bzed/Bmag - bzeta = sqrt(1.0 - bzed^2.0) - Bzeta = Bmag*bzeta - rhostar = get(scan_input, "rhostar", 0.0) - geometry = geometry_input(Bzed,Bmag,bzed,bzeta,Bzeta,rhostar) - - # set composition input - # MRH need to get this in way that does not duplicate code - # MRH from moment_kinetics_input.jl - electron_physics = get(scan_input, "electron_physics", boltzmann_electron_response) - - if electron_physics ∈ (boltzmann_electron_response, boltzmann_electron_response_with_simple_sheath) - n_species = n_ion_species + n_neutral_species - else - n_species = n_ion_species + n_neutral_species + 1 - end - T_e = get(scan_input, "T_e", 1.0) - # set wall temperature T_wall = Tw/Te - T_wall = get(scan_input, "T_wall", 1.0) - # set initial neutral temperature Tn/Tₑ = 1 - # set initial nᵢ/Nₑ = 1.0 - # set phi_wall at z = 0 - phi_wall = get(scan_input, "phi_wall", 0.0) - # if false use true Knudsen cosine for neutral wall bc - use_test_neutral_wall_pdf = get(scan_input, "use_test_neutral_wall_pdf", false) - # constant to be used to test nonzero Er in wall boundary condition - Er_constant = get(scan_input, "Er_constant", 0.0) - recycling_fraction = get(scan_input, "recycling_fraction", 1.0) - # constant to be used to control Ez divergences - epsilon_offset = get(scan_input, "epsilon_offset", 0.001) - # bool to control if dfni is a function of vpa or vpabar in MMS test - use_vpabar_in_mms_dfni = get(scan_input, "use_vpabar_in_mms_dfni", true) - if use_vpabar_in_mms_dfni - alpha_switch = 1.0 - else - alpha_switch = 0.0 - end - # ratio of the neutral particle mass to the ion particle mass - mn_over_mi = 1.0 - # ratio of the electron particle mass to the ion particle mass - me_over_mi = 1.0/1836.0 - composition = species_composition(n_species, n_ion_species, n_neutral_species, - electron_physics, use_test_neutral_wall_pdf, T_e, T_wall, phi_wall, Er_constant, - mn_over_mi, me_over_mi, recycling_fraction, allocate_float(n_species)) - return geometry, composition - -end - """ get_tuple_of_return_values(func, arg_tuples...) @@ -1841,53 +1712,6 @@ function plot_1D_1V_diagnostics(run_name_labels, nwrite_movie, itime_min, itime_ println("done.") end -""" -""" -function calculate_and_write_frequencies(run_name, ntime, time, z, itime_min, itime_max, - iz0, delta_phi, pp) - if pp.calculate_frequencies - println("Calculating the frequency and damping/growth rate...") - # shifted_time = t - t0 - shifted_time = allocate_float(ntime) - @. shifted_time = time - time[itime_min] - # assume phi(z0,t) = A*exp(growth_rate*t)*cos(ω*t + φ) - # and fit phi(z0,t)/phi(z0,t0), which eliminates the constant A pre-factor - @views phi_fit = fit_delta_phi_mode(shifted_time[itime_min:itime_max], z, - delta_phi[:, itime_min:itime_max]) - frequency = phi_fit.frequency - growth_rate = phi_fit.growth_rate - - # write info related to fit to file - io = open_ascii_output_file(run_name, "frequency_fit.txt") - println(io, "#growth_rate: ", phi_fit.growth_rate, - " frequency: ", phi_fit.frequency, - " fit_errors: ", phi_fit.amplitude_fit_error, " ", - phi_fit.offset_fit_error, " ", phi_fit.cosine_fit_error) - println(io) - - # Calculate the fitted phi as a function of time at index iz0 - L = z[end] - z[begin] - fitted_delta_phi = - @. (phi_fit.amplitude0 * cos(2.0 * π * (z[iz0] + phi_fit.offset0) / L) - * exp(phi_fit.growth_rate * shifted_time) - * cos(phi_fit.frequency * shifted_time + phi_fit.phase)) - for i ∈ 1:ntime - println(io, "time: ", time[i], " delta_phi: ", delta_phi[iz0,i], - " fitted_delta_phi: ", fitted_delta_phi[i]) - end - close(io) - else - frequency = 0.0 - growth_rate = 0.0 - phase = 0.0 - shifted_time = allocate_float(ntime) - @. shifted_time = time - time[itime_min] - fitted_delta_phi = zeros(ntime) - - end - return frequency, growth_rate, shifted_time, fitted_delta_phi -end - """ """ function plot_fields(phi, delta_phi, time, itime_min, itime_max, nwrite_movie, @@ -3854,4 +3678,7 @@ function plot_charged_pdf_2D_at_wall(run_name, run_name_label, r_global, z_globa println("done.") end +include("plot_MMS_sequence.jl") +include("plot_sequence.jl") + end diff --git a/src/post_processing_input.jl b/plots_post_processing/src/post_processing_input.jl similarity index 99% rename from src/post_processing_input.jl rename to plots_post_processing/src/post_processing_input.jl index a5e660633..5c3b6f58d 100644 --- a/src/post_processing_input.jl +++ b/plots_post_processing/src/post_processing_input.jl @@ -5,8 +5,8 @@ module post_processing_input export load_post_processing_options export pp -using ..type_definitions: mk_int -using ..input_structs: pp_input +using moment_kinetics.type_definitions: mk_int +using moment_kinetics.input_structs: pp_input # if calculate_frequencies = true, calculate and print the frequency and growth/decay # rate of phi, using values at iz = iz0 diff --git a/plots_post_processing/src/shared_utils.jl b/plots_post_processing/src/shared_utils.jl new file mode 120000 index 000000000..18ff3cde6 --- /dev/null +++ b/plots_post_processing/src/shared_utils.jl @@ -0,0 +1 @@ +../../makie_post_processing/src/shared_utils.jl \ No newline at end of file diff --git a/src/fokker_planck_test.jl b/src/fokker_planck_test.jl index b0391a087..6fa59b84e 100644 --- a/src/fokker_planck_test.jl +++ b/src/fokker_planck_test.jl @@ -14,10 +14,10 @@ export H_Maxwellian, G_Maxwellian export Cssp_fully_expanded_form, calculate_collisional_fluxes -export print_test_data, plot_test_data, fkpl_error_data, allocate_error_data +export print_test_data, fkpl_error_data, allocate_error_data #, plot_test_data -using Plots -using LaTeXStrings +#using Plots +#using LaTeXStrings using Measures using ..type_definitions: mk_float, mk_int using SpecialFunctions: erf @@ -276,21 +276,23 @@ end Below are functions which are used for storing and printing data from the tests """ -function plot_test_data(func_exact,func_num,func_err,func_name,vpa,vperp) - @views heatmap(vperp.grid, vpa.grid, func_num[:,:], ylabel=L"v_{\|\|}", xlabel=L"v_{\perp}", c = :deep, interpolation = :cubic, - windowsize = (360,240), margin = 15pt) - outfile = string(func_name*"_num.pdf") - savefig(outfile) - @views heatmap(vperp.grid, vpa.grid, func_exact[:,:], ylabel=L"v_{\|\|}", xlabel=L"v_{\perp}", c = :deep, interpolation = :cubic, - windowsize = (360,240), margin = 15pt) - outfile = string(func_name*"_exact.pdf") - savefig(outfile) - @views heatmap(vperp.grid, vpa.grid, func_err[:,:], ylabel=L"v_{\|\|}", xlabel=L"v_{\perp}", c = :deep, interpolation = :cubic, - windowsize = (360,240), margin = 15pt) - outfile = string(func_name*"_err.pdf") - savefig(outfile) - return nothing -end +# This function would need to be moved to the plots_post_processing package somewhere, to +# avoid the need for `using Plots` in `moment_kinetics`. +#function plot_test_data(func_exact,func_num,func_err,func_name,vpa,vperp) +# @views heatmap(vperp.grid, vpa.grid, func_num[:,:], ylabel=L"v_{\|\|}", xlabel=L"v_{\perp}", c = :deep, interpolation = :cubic, +# windowsize = (360,240), margin = 15pt) +# outfile = string(func_name*"_num.pdf") +# savefig(outfile) +# @views heatmap(vperp.grid, vpa.grid, func_exact[:,:], ylabel=L"v_{\|\|}", xlabel=L"v_{\perp}", c = :deep, interpolation = :cubic, +# windowsize = (360,240), margin = 15pt) +# outfile = string(func_name*"_exact.pdf") +# savefig(outfile) +# @views heatmap(vperp.grid, vpa.grid, func_err[:,:], ylabel=L"v_{\|\|}", xlabel=L"v_{\perp}", c = :deep, interpolation = :cubic, +# windowsize = (360,240), margin = 15pt) +# outfile = string(func_name*"_err.pdf") +# savefig(outfile) +# return nothing +#end function print_test_data(func_exact,func_num,func_err,func_name) @. func_err = abs(func_num - func_exact) diff --git a/src/moment_kinetics.jl b/src/moment_kinetics.jl index a09a68bbb..32178039a 100644 --- a/src/moment_kinetics.jl +++ b/src/moment_kinetics.jl @@ -62,11 +62,6 @@ include("load_data.jl") include("moment_kinetics_input.jl") include("parameter_scans.jl") include("analysis.jl") -include("post_processing_input.jl") -include("post_processing.jl") -include("plot_MMS_sequence.jl") -include("plot_sequence.jl") -include("makie_post_processing.jl") include("utils.jl") include("time_advance.jl") From 8aac9d49fb0c8edfd1cd5400d57247af3c4beaf3 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 16 Dec 2023 18:10:47 +0000 Subject: [PATCH 02/80] Remove runtime plots It is inconvenient to provide the option for runtime plots when splitting the post-processing functionality into separate packages. The option was not used much anyway. It should be possible to add an 'extension' (with Julia-1.9.x) that provides the option if a post-processing module is loaded, if we want to add it back in future. --- src/input_structs.jl | 1 - src/moment_kinetics_input.jl | 4 +- src/time_advance.jl | 98 ------------------------------------ 3 files changed, 1 insertion(+), 102 deletions(-) diff --git a/src/input_structs.jl b/src/input_structs.jl index 50c79486d..6e51c24b3 100644 --- a/src/input_structs.jl +++ b/src/input_structs.jl @@ -40,7 +40,6 @@ struct time_input nwrite_dfns::mk_int n_rk_stages::mk_int split_operators::Bool - runtime_plots::Bool steady_state_residual::Bool converged_residual_value::mk_float use_manufactured_solns_for_advance::Bool diff --git a/src/moment_kinetics_input.jl b/src/moment_kinetics_input.jl index 1164f22cb..586bbbe9b 100644 --- a/src/moment_kinetics_input.jl +++ b/src/moment_kinetics_input.jl @@ -197,7 +197,6 @@ function mk_input(scan_input=Dict(); save_inputs_to_txt=false, ignore_MPI=true) # Heun's method, SSP RK3 and 4-stage SSP RK3) n_rk_stages = get(scan_input, "n_rk_stages", 4) split_operators = get(scan_input, "split_operators", false) - runtime_plots = get(scan_input, "runtime_plots", false) stopfile_name = joinpath(output_dir, "stop") steady_state_residual = get(scan_input, "steady_state_residual", false) converged_residual_value = get(scan_input, "converged_residual_value", -1.0) @@ -394,8 +393,7 @@ function mk_input(scan_input=Dict(); save_inputs_to_txt=false, ignore_MPI=true) #nrank_z = 0 t_input = time_input(nstep, dt, nwrite_moments, nwrite_dfns, n_rk_stages, - split_operators, runtime_plots, steady_state_residual, - converged_residual_value, + split_operators, steady_state_residual, converged_residual_value, manufactured_solns_input.use_for_advance, stopfile_name) # replace mutable structures with immutable ones to optimize performance # and avoid possible misunderstandings diff --git a/src/time_advance.jl b/src/time_advance.jl index 109e94ebb..4e27925e2 100644 --- a/src/time_advance.jl +++ b/src/time_advance.jl @@ -28,7 +28,6 @@ using ..velocity_grid_transforms: vzvrvzeta_to_vpavperp!, vpavperp_to_vzvrvzeta! using ..initial_conditions: enforce_boundary_conditions! using ..initial_conditions: enforce_neutral_boundary_conditions! using ..input_structs: advance_info, time_input -using ..makie_post_processing: plot_1d, plot_2d, positive_or_nan using ..moment_constraints: hard_force_moment_constraints!, hard_force_moment_constraints_neutral! using ..advection: setup_advection @@ -62,7 +61,6 @@ using ..utils: to_minutes @debug_detect_redundant_block_synchronize using ..communication: debug_detect_redundant_is_active using Dates -using CairoMakie using ..analysis: steady_state_residuals #using ..post_processing: draw_v_parallel_zero! @@ -983,102 +981,6 @@ function time_advance!(pdf, scratch, t, t_input, vz, vr, vzeta, vpa, vperp, gyro end end - # Hack to save *.pdf of current pdf - if t_input.runtime_plots - if block_rank[] == 0 - fig = Figure() - - irow = 1 - title_layout = fig[irow,1] = GridLayout() - Label(title_layout[1,1:2], string(t)) - - ax_width = 400 - ax_height = 400 - - irow += 1 - layout = fig[irow,1] = GridLayout() - ax = Axis(layout[1,1], xlabel="vpa", ylabel="z", title="f", width=ax_width, height=ax_height) - plot_2d(vpa.grid, z.grid, pdf.charged.norm[:,1,:,1,1]; ax=ax, colorbar_place=layout[1,2]) - if composition.n_neutral_species > 0 - ax = Axis(layout[1,3], xlabel="vz", ylabel="z", title="f_neutral", width=ax_width, height=ax_height) - plot_2d(vz.grid, z.grid, pdf.neutral.norm[:,1,1,:,1,1]; ax=ax, colorbar_place=layout[1,4]) - end - - irow += 1 - layout = fig[irow,1] = GridLayout() - ax = Axis(layout[1,1], xlabel="vpa", ylabel="z", title="f", width=ax_width, height=ax_height) - plot_2d(vpa.grid, z.grid, pdf.charged.norm[:,1,:,1,1]; ax=ax, - colorbar_place=layout[1,2], colorscale=log10, - transform=x->positive_or_nan(x, epsilon=1.e-20)) - if composition.n_neutral_species > 0 - ax = Axis(layout[1,3], xlabel="vz", ylabel="z", title="f_neutral", width=ax_width, height=ax_height) - plot_2d(vz.grid, z.grid, pdf.neutral.norm[:,1,1,:,1,1]; ax=ax, - colorbar_place=layout[1,4], colorscale=log10, - transform=x->positive_or_nan(x, epsilon=1.e-20)) - end - - irow += 1 - layout = fig[irow,1] = GridLayout() - ax = Axis(layout[1,1], xlabel="vpa", ylabel="f0", width=ax_width, height=ax_height) - plot_1d(vpa.grid, pdf.charged.norm[:,1,1,1,1]; ax=ax) - if composition.n_neutral_species > 0 - ax = Axis(layout[1,2], xlabel="vz", ylabel="f0_neutral", width=ax_width, height=ax_height) - plot_1d(vz.grid, pdf.neutral.norm[:,1,1,1,1,1]; ax=ax) - end - - irow += 1 - layout = fig[irow,1] = GridLayout() - ax = Axis(layout[1,1], xlabel="vpa", ylabel="fL", width=ax_width, height=ax_height) - plot_1d(vpa.grid, pdf.charged.norm[:,1,end,1,1]; ax=ax) - if composition.n_neutral_species > 0 - ax = Axis(layout[1,2], xlabel="vz", ylabel="fL_neutral", width=ax_width, height=ax_height) - plot_1d(vz.grid, pdf.neutral.norm[:,1,1,end,1,1]; ax=ax) - end - - irow += 1 - layout = fig[irow,1] = GridLayout() - ax = Axis(layout[1,1], xlabel="z", ylabel="density", width=ax_width, height=ax_height) - plot_1d(z.grid, moments.charged.dens[:,1,1]; ax=ax, label="ion") - if composition.n_neutral_species > 0 - plot_1d(z.grid, moments.neutral.dens[:,1,1]; ax=ax, label="neutral") - end - #axislegend(ax) - ax = Axis(layout[1,2], xlabel="z", ylabel="upar", width=ax_width, height=ax_height) - plot_1d(z.grid, moments.charged.upar[:,1,1]; ax=ax, label="ion") - if composition.n_neutral_species > 0 - plot_1d(z.grid, moments.neutral.uz[:,1,1]; ax=ax, label="neutral") - end - #axislegend(ax) - - irow += 1 - layout = fig[irow,1] = GridLayout() - ax = Axis(layout[1,1], xlabel="z", ylabel="ppar", width=ax_width, height=ax_height) - plot_1d(z.grid, moments.charged.ppar[:,1,1]; ax=ax, label="ion") - if composition.n_neutral_species > 0 - plot_1d(z.grid, moments.neutral.pz[:,1,1]; ax=ax, label="neutral") - end - #axislegend(ax) - ax = Axis(layout[1,2], xlabel="z", ylabel="vth", width=ax_width, height=ax_height) - plot_1d(z.grid, moments.charged.vth[:,1,1]; ax=ax, label="ion") - if composition.n_neutral_species > 0 - plot_1d(z.grid, moments.neutral.vth[:,1,1]; ax=ax, label="neutral") - end - #axislegend(ax) - - irow += 1 - layout = fig[irow,1] = GridLayout() - ax = Axis(layout[1,1], xlabel="z", ylabel="qpar", width=ax_width, height=ax_height) - plot_1d(z.grid, moments.charged.qpar[:,1,1]; ax=ax, label="ion") - if composition.n_neutral_species > 0 - plot_1d(z.grid, moments.neutral.qz[:,1,1]; ax=ax, label="neutral") - end - #axislegend(ax) - - resize_to_layout!(fig) - - save("latest_plots$(iblock_index[]).png", fig) - end - end iwrite_moments += 1 begin_s_r_z_vperp_region() @debug_detect_redundant_block_synchronize begin From 9fa67d802ec22f22e63aaeb6b82724100e3e2cb4 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 16 Dec 2023 18:27:46 +0000 Subject: [PATCH 03/80] Support new post-processing packages in Documenter.jl setup --- .github/workflows/documentation.yml | 2 +- docs/make.jl | 2 +- docs/src/zz_makie_post_processing.md | 2 +- docs/src/zz_plots_post_processing.md | 6 ++++++ docs/src/zz_post_processing.md | 6 ------ docs/src/zz_post_processing_input.md | 6 ------ 6 files changed, 9 insertions(+), 15 deletions(-) create mode 100644 docs/src/zz_plots_post_processing.md delete mode 100644 docs/src/zz_post_processing.md delete mode 100644 docs/src/zz_post_processing_input.md diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index c7a611d9a..ea1f941b1 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -18,7 +18,7 @@ jobs: - name: Install dependencies run: | pip3 install --user matplotlib - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()' + julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.develop(PackageSpec(path=joinpath(pwd(), "makie_post_processing"))); Pkg.develop(PackageSpec(path=joinpath(pwd(), "plots_post_processing"))); Pkg.instantiate()' - name: Build and deploy env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Authenticate with GitHub Actions token diff --git a/docs/make.jl b/docs/make.jl index 5c415c2c0..2c685b6af 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -4,7 +4,7 @@ using moment_kinetics makedocs( sitename = "moment_kinetics", format = Documenter.HTML(prettyurls = get(ENV, "CI", nothing) == "true"), - modules = [moment_kinetics] + modules = [moment_kinetics, makie_post_processing, plots_post_processing] ) if get(ENV, "CI", nothing) == "true" diff --git a/docs/src/zz_makie_post_processing.md b/docs/src/zz_makie_post_processing.md index d2ad434cb..ef0ceed37 100644 --- a/docs/src/zz_makie_post_processing.md +++ b/docs/src/zz_makie_post_processing.md @@ -2,5 +2,5 @@ ======================= ```@autodocs -Modules = [moment_kinetics.makie_post_processing] +Modules = [makie_post_processing] ``` diff --git a/docs/src/zz_plots_post_processing.md b/docs/src/zz_plots_post_processing.md new file mode 100644 index 000000000..89743c4fd --- /dev/null +++ b/docs/src/zz_plots_post_processing.md @@ -0,0 +1,6 @@ +`plots_post_processing` +======================= + +```@autodocs +Modules = [plots_post_processing, plots_post_processing.post_processing_input] +``` diff --git a/docs/src/zz_post_processing.md b/docs/src/zz_post_processing.md deleted file mode 100644 index ab0d10409..000000000 --- a/docs/src/zz_post_processing.md +++ /dev/null @@ -1,6 +0,0 @@ -`post_processing` -================= - -```@autodocs -Modules = [moment_kinetics.post_processing] -``` diff --git a/docs/src/zz_post_processing_input.md b/docs/src/zz_post_processing_input.md deleted file mode 100644 index 67e13a54d..000000000 --- a/docs/src/zz_post_processing_input.md +++ /dev/null @@ -1,6 +0,0 @@ -`post_processing_input` -======================= - -```@autodocs -Modules = [moment_kinetics.post_processing_input] -``` From cd5748332abef4f71d8fd920b2e6a1c7805a19df Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 17 Dec 2023 11:54:42 +0000 Subject: [PATCH 04/80] Update docs to reflect restructured post-processing packages --- docs/Project.toml | 2 ++ docs/make.jl | 2 +- docs/src/parameter_scans.md | 6 ++--- docs/src/post_processing_notes.md | 33 ++++++++++++++-------------- docs/src/zz_makie_post_processing.md | 2 +- docs/src/zz_plot_MMS_sequence.md | 2 +- docs/src/zz_plot_sequence.md | 2 +- docs/src/zz_plots_post_processing.md | 2 +- 8 files changed, 26 insertions(+), 25 deletions(-) diff --git a/docs/Project.toml b/docs/Project.toml index 10ee60ca2..ab9ea9a8f 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,6 +1,8 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +makie_post_processing = "14778871-8d0d-429c-ae6e-bde7d57d7f1e" moment_kinetics = "b5ff72cc-06fc-4161-ad14-dba1c22ed34e" +plots_post_processing = "6f98f6d0-83ae-4c2a-9d92-db6b25a4b00d" [compat] Documenter = "0.27" diff --git a/docs/make.jl b/docs/make.jl index 2c685b6af..5a5130564 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,5 +1,5 @@ using Documenter -using moment_kinetics +using moment_kinetics, makie_post_processing, plots_post_processing makedocs( sitename = "moment_kinetics", diff --git a/docs/src/parameter_scans.md b/docs/src/parameter_scans.md index b828c4e5f..6b8eff9ec 100644 --- a/docs/src/parameter_scans.md +++ b/docs/src/parameter_scans.md @@ -35,9 +35,9 @@ every input file contained in that directory. Post processing a scan ---------------------- -[`moment_kinetics.makie_post_processing.makie_post_process`](@ref) can be -called for each run in a scan. For example to post process the scan in -`runs/scan_example` from the REPL +[`makie_post_processing.makie_post_process`](@ref) can be called for each run +in a scan. For example to post process the scan in `runs/scan_example` from the +REPL ```julia $ julia -p 8 --project -O3 julia> include("post_process_parameter_scan.jl") diff --git a/docs/src/post_processing_notes.md b/docs/src/post_processing_notes.md index 8633ded59..d78380a44 100644 --- a/docs/src/post_processing_notes.md +++ b/docs/src/post_processing_notes.md @@ -5,11 +5,10 @@ How to ------ Post processing functionality is provided by the -[`moment_kinetics.makie_post_processing`](@ref) module. To run the post -processing, call -[`moment_kinetics.makie_post_processing.makie_post_process`](@ref) e.g. +[`makie_post_processing.makie_post_processing`](@ref) module. To run the post +processing, call [`makie_post_processing.makie_post_process`](@ref) e.g. ```julia -julia> using moment_kinetics.makie_post_processing +julia> using makie_post_processing julia> makie_post_process("runs/example-run/") ``` or @@ -35,9 +34,9 @@ default they will all be read and plotted. A single restart can be started by passing the `restart_id` argument to `makie_post_process()`. To see all the options that can be set, -[`moment_kinetics.makie_post_processing.generate_example_input_file`](@ref) can -be used to create an example file containing all the options with their default -values. The options are all commented out when the file is created. +[`makie_post_processing.generate_example_input_file`](@ref) can be used to +create an example file containing all the options with their default values. +The options are all commented out when the file is created. !!! note "Viewing animations" Animations are produced in `.gif` format. Most utilities just play gifs, @@ -48,12 +47,12 @@ values. The options are all commented out when the file is created. Interactive use --------------- -The functions in [`moment_kinetics.makie_post_processing`](@ref) can be used -interactively (or in standalone scripts). To do so, first get the 'info' for -a run (file names, metadata, etc.) using -[`moment_kinetics.makie_post_processing.get_run_info`](@ref) +The functions in [`makie_post_processing.makie_post_processing`](@ref) can be +used interactively (or in standalone scripts). To do so, first get the 'info' +for a run (file names, metadata, etc.) using +[`makie_post_processing.get_run_info`](@ref) ```julia -julia> using moment_kinetics.makie_post_processing +julia> using makie_post_processing julia> run_info = get_run_info("runs/example-run/") ``` or to load from the distribution functions output file `.dfns.h5` @@ -75,8 +74,8 @@ Then you can make 1d or 2d plots, e.g. julia> fig1 = plot_vs_z(run_info, "phi") julia> fig2 = plot_vs_r_t(run_info, "density"; outfile="density_vs_r_t.pdf") ``` -using [`moment_kinetics.makie_post_processing.plot_vs_t`](@ref), etc. for 1d -and [`moment_kinetics.makie_post_processing.plot_vs_r_t`](@ref), etc. for 2d +using [`makie_post_processing.plot_vs_t`](@ref), etc. for 1d and +[`makie_post_processing.plot_vs_r_t`](@ref), etc. for 2d plots. The `outfile` argument can be used to save the plot. You can also change the default values used to select from the other dimensions @@ -88,9 +87,9 @@ You can make animations in a similar way julia> fig1 = animate_vs_z(run_info, "phi"; outfile="phi_vs_z.gif", it=8:12, ir=1) julia> fig2 = animate_vs_z_r(run_info, "density"; outfile="density_vs_z_r.mp4") ``` -using [`moment_kinetics.makie_post_processing.animate_vs_r`](@ref), etc. for 1d -and [`moment_kinetics.makie_post_processing.animate_vs_z_r`](@ref), etc. for -2d animations. +using [`makie_post_processing.animate_vs_r`](@ref), etc. for 1d and +[`makie_post_processing.animate_vs_z_r`](@ref), etc. for 2d +animations. Note that `outfile` is required for animations. API diff --git a/docs/src/zz_makie_post_processing.md b/docs/src/zz_makie_post_processing.md index ef0ceed37..6752d3e69 100644 --- a/docs/src/zz_makie_post_processing.md +++ b/docs/src/zz_makie_post_processing.md @@ -2,5 +2,5 @@ ======================= ```@autodocs -Modules = [makie_post_processing] +Modules = [makie_post_processing, makie_post_processing.shared_utils] ``` diff --git a/docs/src/zz_plot_MMS_sequence.md b/docs/src/zz_plot_MMS_sequence.md index adb44f00d..f6b446cb9 100644 --- a/docs/src/zz_plot_MMS_sequence.md +++ b/docs/src/zz_plot_MMS_sequence.md @@ -2,5 +2,5 @@ =================== ```@autodocs -Modules = [moment_kinetics.plot_MMS_sequence] +Modules = [plots_post_processing.plot_MMS_sequence] ``` diff --git a/docs/src/zz_plot_sequence.md b/docs/src/zz_plot_sequence.md index 16b1d9ca2..e8eccddf6 100644 --- a/docs/src/zz_plot_sequence.md +++ b/docs/src/zz_plot_sequence.md @@ -2,5 +2,5 @@ =============== ```@autodocs -Modules = [moment_kinetics.plot_sequence] +Modules = [plots_post_processing.plot_sequence] ``` diff --git a/docs/src/zz_plots_post_processing.md b/docs/src/zz_plots_post_processing.md index 89743c4fd..1f2d4d9f2 100644 --- a/docs/src/zz_plots_post_processing.md +++ b/docs/src/zz_plots_post_processing.md @@ -2,5 +2,5 @@ ======================= ```@autodocs -Modules = [plots_post_processing, plots_post_processing.post_processing_input] +Modules = [plots_post_processing, plots_post_processing.post_processing_input, plots_post_processing.shared_utils] ``` From 007c9fbe666b63016958a5a81b46af569e22e206 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 19 Dec 2023 14:09:46 +0000 Subject: [PATCH 05/80] Move moment_kinetics package to a subdirectory Allows us to separate the Project.toml for the moment_kinetics package from the Project.toml that a user sets up to run the code. --- .gitignore | 3 ++- Project.toml => moment_kinetics/Project.toml | 1 - {debug_test => moment_kinetics/debug_test}/README.md | 0 .../debug_redundant_synchronization/harrisonthompson.jl | 0 .../debug_redundant_synchronization/runtest_template.jl | 0 .../debug_test}/debug_redundant_synchronization/runtests.jl | 0 .../debug_redundant_synchronization/sound_wave_tests.jl | 0 .../debug_redundant_synchronization/wall_bc_tests.jl | 0 .../debug_test}/fokker_planck_collisions_inputs.jl | 0 .../debug_test}/fokker_planck_collisions_tests.jl | 0 {debug_test => moment_kinetics/debug_test}/harrisonthompson.jl | 0 .../debug_test}/harrisonthompson_inputs.jl | 0 {debug_test => moment_kinetics/debug_test}/mms_inputs.jl | 0 {debug_test => moment_kinetics/debug_test}/mms_tests.jl | 0 .../debug_test}/restart_interpolation_inputs.jl | 0 .../debug_test}/restart_interpolation_tests.jl | 0 {debug_test => moment_kinetics/debug_test}/runtest_template.jl | 0 {debug_test => moment_kinetics/debug_test}/runtests.jl | 0 {debug_test => moment_kinetics/debug_test}/setup.jl | 0 .../debug_test}/sound_wave_inputs.jl | 0 {debug_test => moment_kinetics/debug_test}/sound_wave_tests.jl | 0 {debug_test => moment_kinetics/debug_test}/wall_bc_inputs.jl | 0 {debug_test => moment_kinetics/debug_test}/wall_bc_tests.jl | 0 {src => moment_kinetics/src}/advection.jl | 0 {src => moment_kinetics/src}/analysis.jl | 0 {src => moment_kinetics/src}/array_allocation.jl | 0 {src => moment_kinetics/src}/bgk.jl | 0 {src => moment_kinetics/src}/calculus.jl | 0 {src => moment_kinetics/src}/charge_exchange.jl | 0 {src => moment_kinetics/src}/chebyshev.jl | 0 {src => moment_kinetics/src}/clenshaw_curtis.jl | 0 {src => moment_kinetics/src}/command_line_options.jl | 0 {src => moment_kinetics/src}/communication.jl | 0 {src => moment_kinetics/src}/constants.jl | 0 {src => moment_kinetics/src}/continuity.jl | 0 {src => moment_kinetics/src}/coordinates.jl | 0 {src => moment_kinetics/src}/debugging.jl | 0 {src => moment_kinetics/src}/derivatives.jl | 0 {src => moment_kinetics/src}/em_fields.jl | 0 {src => moment_kinetics/src}/energy_equation.jl | 0 {src => moment_kinetics/src}/external_sources.jl | 0 {src => moment_kinetics/src}/file_io.jl | 2 +- {src => moment_kinetics/src}/file_io_hdf5.jl | 0 {src => moment_kinetics/src}/file_io_netcdf.jl | 0 {src => moment_kinetics/src}/finite_differences.jl | 0 {src => moment_kinetics/src}/fokker_planck.jl | 0 {src => moment_kinetics/src}/fokker_planck_calculus.jl | 0 {src => moment_kinetics/src}/fokker_planck_test.jl | 0 {src => moment_kinetics/src}/force_balance.jl | 0 {src => moment_kinetics/src}/gauss_legendre.jl | 0 {src => moment_kinetics/src}/hermite_spline_interpolation.jl | 0 {src => moment_kinetics/src}/initial_conditions.jl | 0 {src => moment_kinetics/src}/input_structs.jl | 0 {src => moment_kinetics/src}/interpolation.jl | 0 {src => moment_kinetics/src}/ionization.jl | 0 {src => moment_kinetics/src}/krook_collisions.jl | 0 {src => moment_kinetics/src}/load_data.jl | 0 {src => moment_kinetics/src}/looping.jl | 0 {src => moment_kinetics/src}/manufactured_solns.jl | 0 {src => moment_kinetics/src}/moment_constraints.jl | 0 {src => moment_kinetics/src}/moment_kinetics.jl | 2 +- {src => moment_kinetics/src}/moment_kinetics_input.jl | 0 {src => moment_kinetics/src}/moment_kinetics_structs.jl | 0 {src => moment_kinetics/src}/neutral_advection.jl | 0 {src => moment_kinetics/src}/neutral_r_advection.jl | 0 {src => moment_kinetics/src}/neutral_vz_advection.jl | 0 {src => moment_kinetics/src}/neutral_z_advection.jl | 0 {src => moment_kinetics/src}/numerical_dissipation.jl | 0 {src => moment_kinetics/src}/parameter_scans.jl | 0 {src => moment_kinetics/src}/quadrature.jl | 0 {src => moment_kinetics/src}/r_advection.jl | 0 {src => moment_kinetics/src}/reference_parameters.jl | 0 {src => moment_kinetics/src}/source_terms.jl | 0 {src => moment_kinetics/src}/time_advance.jl | 0 {src => moment_kinetics/src}/type_definitions.jl | 0 {src => moment_kinetics/src}/utils.jl | 0 {src => moment_kinetics/src}/velocity_grid_transforms.jl | 0 {src => moment_kinetics/src}/velocity_moments.jl | 0 {src => moment_kinetics/src}/vpa_advection.jl | 0 {src => moment_kinetics/src}/vperp_advection.jl | 0 {src => moment_kinetics/src}/z_advection.jl | 0 {test => moment_kinetics/test}/Krook_collisions_tests.jl | 0 {test => moment_kinetics/test}/Project.toml | 0 {test => moment_kinetics/test}/calculus_tests.jl | 0 {test => moment_kinetics/test}/fokker_planck_tests.jl | 0 .../test}/fokker_planck_time_evolution_tests.jl | 0 {test => moment_kinetics/test}/harrisonthompson.jl | 0 {test => moment_kinetics/test}/interpolation_tests.jl | 0 {test => moment_kinetics/test}/loop_setup_tests.jl | 0 .../test}/nonlinear_sound_wave_inputs_and_expected_data.jl | 0 {test => moment_kinetics/test}/nonlinear_sound_wave_tests.jl | 0 {test => moment_kinetics/test}/recycling_fraction_tests.jl | 0 {test => moment_kinetics/test}/restart_interpolation_tests.jl | 0 {test => moment_kinetics/test}/runtests.jl | 0 {test => moment_kinetics/test}/setup.jl | 0 {test => moment_kinetics/test}/sound_wave_tests.jl | 0 {test => moment_kinetics/test}/velocity_integral_tests.jl | 0 {test => moment_kinetics/test}/wall_bc_tests.jl | 0 98 files changed, 4 insertions(+), 4 deletions(-) rename Project.toml => moment_kinetics/Project.toml (97%) rename {debug_test => moment_kinetics/debug_test}/README.md (100%) rename {debug_test => moment_kinetics/debug_test}/debug_redundant_synchronization/harrisonthompson.jl (100%) rename {debug_test => moment_kinetics/debug_test}/debug_redundant_synchronization/runtest_template.jl (100%) rename {debug_test => moment_kinetics/debug_test}/debug_redundant_synchronization/runtests.jl (100%) rename {debug_test => moment_kinetics/debug_test}/debug_redundant_synchronization/sound_wave_tests.jl (100%) rename {debug_test => moment_kinetics/debug_test}/debug_redundant_synchronization/wall_bc_tests.jl (100%) rename {debug_test => moment_kinetics/debug_test}/fokker_planck_collisions_inputs.jl (100%) rename {debug_test => moment_kinetics/debug_test}/fokker_planck_collisions_tests.jl (100%) rename {debug_test => moment_kinetics/debug_test}/harrisonthompson.jl (100%) rename {debug_test => moment_kinetics/debug_test}/harrisonthompson_inputs.jl (100%) rename {debug_test => moment_kinetics/debug_test}/mms_inputs.jl (100%) rename {debug_test => moment_kinetics/debug_test}/mms_tests.jl (100%) rename {debug_test => moment_kinetics/debug_test}/restart_interpolation_inputs.jl (100%) rename {debug_test => moment_kinetics/debug_test}/restart_interpolation_tests.jl (100%) rename {debug_test => moment_kinetics/debug_test}/runtest_template.jl (100%) rename {debug_test => moment_kinetics/debug_test}/runtests.jl (100%) rename {debug_test => moment_kinetics/debug_test}/setup.jl (100%) rename {debug_test => moment_kinetics/debug_test}/sound_wave_inputs.jl (100%) rename {debug_test => moment_kinetics/debug_test}/sound_wave_tests.jl (100%) rename {debug_test => moment_kinetics/debug_test}/wall_bc_inputs.jl (100%) rename {debug_test => moment_kinetics/debug_test}/wall_bc_tests.jl (100%) rename {src => moment_kinetics/src}/advection.jl (100%) rename {src => moment_kinetics/src}/analysis.jl (100%) rename {src => moment_kinetics/src}/array_allocation.jl (100%) rename {src => moment_kinetics/src}/bgk.jl (100%) rename {src => moment_kinetics/src}/calculus.jl (100%) rename {src => moment_kinetics/src}/charge_exchange.jl (100%) rename {src => moment_kinetics/src}/chebyshev.jl (100%) rename {src => moment_kinetics/src}/clenshaw_curtis.jl (100%) rename {src => moment_kinetics/src}/command_line_options.jl (100%) rename {src => moment_kinetics/src}/communication.jl (100%) rename {src => moment_kinetics/src}/constants.jl (100%) rename {src => moment_kinetics/src}/continuity.jl (100%) rename {src => moment_kinetics/src}/coordinates.jl (100%) rename {src => moment_kinetics/src}/debugging.jl (100%) rename {src => moment_kinetics/src}/derivatives.jl (100%) rename {src => moment_kinetics/src}/em_fields.jl (100%) rename {src => moment_kinetics/src}/energy_equation.jl (100%) rename {src => moment_kinetics/src}/external_sources.jl (100%) rename {src => moment_kinetics/src}/file_io.jl (99%) rename {src => moment_kinetics/src}/file_io_hdf5.jl (100%) rename {src => moment_kinetics/src}/file_io_netcdf.jl (100%) rename {src => moment_kinetics/src}/finite_differences.jl (100%) rename {src => moment_kinetics/src}/fokker_planck.jl (100%) rename {src => moment_kinetics/src}/fokker_planck_calculus.jl (100%) rename {src => moment_kinetics/src}/fokker_planck_test.jl (100%) rename {src => moment_kinetics/src}/force_balance.jl (100%) rename {src => moment_kinetics/src}/gauss_legendre.jl (100%) rename {src => moment_kinetics/src}/hermite_spline_interpolation.jl (100%) rename {src => moment_kinetics/src}/initial_conditions.jl (100%) rename {src => moment_kinetics/src}/input_structs.jl (100%) rename {src => moment_kinetics/src}/interpolation.jl (100%) rename {src => moment_kinetics/src}/ionization.jl (100%) rename {src => moment_kinetics/src}/krook_collisions.jl (100%) rename {src => moment_kinetics/src}/load_data.jl (100%) rename {src => moment_kinetics/src}/looping.jl (100%) rename {src => moment_kinetics/src}/manufactured_solns.jl (100%) rename {src => moment_kinetics/src}/moment_constraints.jl (100%) rename {src => moment_kinetics/src}/moment_kinetics.jl (99%) rename {src => moment_kinetics/src}/moment_kinetics_input.jl (100%) rename {src => moment_kinetics/src}/moment_kinetics_structs.jl (100%) rename {src => moment_kinetics/src}/neutral_advection.jl (100%) rename {src => moment_kinetics/src}/neutral_r_advection.jl (100%) rename {src => moment_kinetics/src}/neutral_vz_advection.jl (100%) rename {src => moment_kinetics/src}/neutral_z_advection.jl (100%) rename {src => moment_kinetics/src}/numerical_dissipation.jl (100%) rename {src => moment_kinetics/src}/parameter_scans.jl (100%) rename {src => moment_kinetics/src}/quadrature.jl (100%) rename {src => moment_kinetics/src}/r_advection.jl (100%) rename {src => moment_kinetics/src}/reference_parameters.jl (100%) rename {src => moment_kinetics/src}/source_terms.jl (100%) rename {src => moment_kinetics/src}/time_advance.jl (100%) rename {src => moment_kinetics/src}/type_definitions.jl (100%) rename {src => moment_kinetics/src}/utils.jl (100%) rename {src => moment_kinetics/src}/velocity_grid_transforms.jl (100%) rename {src => moment_kinetics/src}/velocity_moments.jl (100%) rename {src => moment_kinetics/src}/vpa_advection.jl (100%) rename {src => moment_kinetics/src}/vperp_advection.jl (100%) rename {src => moment_kinetics/src}/z_advection.jl (100%) rename {test => moment_kinetics/test}/Krook_collisions_tests.jl (100%) rename {test => moment_kinetics/test}/Project.toml (100%) rename {test => moment_kinetics/test}/calculus_tests.jl (100%) rename {test => moment_kinetics/test}/fokker_planck_tests.jl (100%) rename {test => moment_kinetics/test}/fokker_planck_time_evolution_tests.jl (100%) rename {test => moment_kinetics/test}/harrisonthompson.jl (100%) rename {test => moment_kinetics/test}/interpolation_tests.jl (100%) rename {test => moment_kinetics/test}/loop_setup_tests.jl (100%) rename {test => moment_kinetics/test}/nonlinear_sound_wave_inputs_and_expected_data.jl (100%) rename {test => moment_kinetics/test}/nonlinear_sound_wave_tests.jl (100%) rename {test => moment_kinetics/test}/recycling_fraction_tests.jl (100%) rename {test => moment_kinetics/test}/restart_interpolation_tests.jl (100%) rename {test => moment_kinetics/test}/runtests.jl (100%) rename {test => moment_kinetics/test}/setup.jl (100%) rename {test => moment_kinetics/test}/sound_wave_tests.jl (100%) rename {test => moment_kinetics/test}/velocity_integral_tests.jl (100%) rename {test => moment_kinetics/test}/wall_bc_tests.jl (100%) diff --git a/.gitignore b/.gitignore index bc6b56b71..0f458cf2a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ performance-tests/config.toml Manifest.toml LocalPreferences.toml +/Project.toml /julia.env bin machines/shared/compile_dependencies.sh @@ -9,4 +10,4 @@ machines/shared/machine_setup_stage_two.jl machines/artifacts moment_kinetics.so precompile-temp -post_processing_input.toml \ No newline at end of file +post_processing_input.toml diff --git a/Project.toml b/moment_kinetics/Project.toml similarity index 97% rename from Project.toml rename to moment_kinetics/Project.toml index 49535c119..23e5521f1 100644 --- a/Project.toml +++ b/moment_kinetics/Project.toml @@ -22,7 +22,6 @@ MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" Measures = "442fdcdd-2543-5da2-b0f3-8c86c306513e" NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -PackageCompiler = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Preferences = "21216c6a-2e73-6563-6e65-726566657250" Primes = "27ebfcd6-29c5-5fa9-bf4b-fb8fc14df3ae" diff --git a/debug_test/README.md b/moment_kinetics/debug_test/README.md similarity index 100% rename from debug_test/README.md rename to moment_kinetics/debug_test/README.md diff --git a/debug_test/debug_redundant_synchronization/harrisonthompson.jl b/moment_kinetics/debug_test/debug_redundant_synchronization/harrisonthompson.jl similarity index 100% rename from debug_test/debug_redundant_synchronization/harrisonthompson.jl rename to moment_kinetics/debug_test/debug_redundant_synchronization/harrisonthompson.jl diff --git a/debug_test/debug_redundant_synchronization/runtest_template.jl b/moment_kinetics/debug_test/debug_redundant_synchronization/runtest_template.jl similarity index 100% rename from debug_test/debug_redundant_synchronization/runtest_template.jl rename to moment_kinetics/debug_test/debug_redundant_synchronization/runtest_template.jl diff --git a/debug_test/debug_redundant_synchronization/runtests.jl b/moment_kinetics/debug_test/debug_redundant_synchronization/runtests.jl similarity index 100% rename from debug_test/debug_redundant_synchronization/runtests.jl rename to moment_kinetics/debug_test/debug_redundant_synchronization/runtests.jl diff --git a/debug_test/debug_redundant_synchronization/sound_wave_tests.jl b/moment_kinetics/debug_test/debug_redundant_synchronization/sound_wave_tests.jl similarity index 100% rename from debug_test/debug_redundant_synchronization/sound_wave_tests.jl rename to moment_kinetics/debug_test/debug_redundant_synchronization/sound_wave_tests.jl diff --git a/debug_test/debug_redundant_synchronization/wall_bc_tests.jl b/moment_kinetics/debug_test/debug_redundant_synchronization/wall_bc_tests.jl similarity index 100% rename from debug_test/debug_redundant_synchronization/wall_bc_tests.jl rename to moment_kinetics/debug_test/debug_redundant_synchronization/wall_bc_tests.jl diff --git a/debug_test/fokker_planck_collisions_inputs.jl b/moment_kinetics/debug_test/fokker_planck_collisions_inputs.jl similarity index 100% rename from debug_test/fokker_planck_collisions_inputs.jl rename to moment_kinetics/debug_test/fokker_planck_collisions_inputs.jl diff --git a/debug_test/fokker_planck_collisions_tests.jl b/moment_kinetics/debug_test/fokker_planck_collisions_tests.jl similarity index 100% rename from debug_test/fokker_planck_collisions_tests.jl rename to moment_kinetics/debug_test/fokker_planck_collisions_tests.jl diff --git a/debug_test/harrisonthompson.jl b/moment_kinetics/debug_test/harrisonthompson.jl similarity index 100% rename from debug_test/harrisonthompson.jl rename to moment_kinetics/debug_test/harrisonthompson.jl diff --git a/debug_test/harrisonthompson_inputs.jl b/moment_kinetics/debug_test/harrisonthompson_inputs.jl similarity index 100% rename from debug_test/harrisonthompson_inputs.jl rename to moment_kinetics/debug_test/harrisonthompson_inputs.jl diff --git a/debug_test/mms_inputs.jl b/moment_kinetics/debug_test/mms_inputs.jl similarity index 100% rename from debug_test/mms_inputs.jl rename to moment_kinetics/debug_test/mms_inputs.jl diff --git a/debug_test/mms_tests.jl b/moment_kinetics/debug_test/mms_tests.jl similarity index 100% rename from debug_test/mms_tests.jl rename to moment_kinetics/debug_test/mms_tests.jl diff --git a/debug_test/restart_interpolation_inputs.jl b/moment_kinetics/debug_test/restart_interpolation_inputs.jl similarity index 100% rename from debug_test/restart_interpolation_inputs.jl rename to moment_kinetics/debug_test/restart_interpolation_inputs.jl diff --git a/debug_test/restart_interpolation_tests.jl b/moment_kinetics/debug_test/restart_interpolation_tests.jl similarity index 100% rename from debug_test/restart_interpolation_tests.jl rename to moment_kinetics/debug_test/restart_interpolation_tests.jl diff --git a/debug_test/runtest_template.jl b/moment_kinetics/debug_test/runtest_template.jl similarity index 100% rename from debug_test/runtest_template.jl rename to moment_kinetics/debug_test/runtest_template.jl diff --git a/debug_test/runtests.jl b/moment_kinetics/debug_test/runtests.jl similarity index 100% rename from debug_test/runtests.jl rename to moment_kinetics/debug_test/runtests.jl diff --git a/debug_test/setup.jl b/moment_kinetics/debug_test/setup.jl similarity index 100% rename from debug_test/setup.jl rename to moment_kinetics/debug_test/setup.jl diff --git a/debug_test/sound_wave_inputs.jl b/moment_kinetics/debug_test/sound_wave_inputs.jl similarity index 100% rename from debug_test/sound_wave_inputs.jl rename to moment_kinetics/debug_test/sound_wave_inputs.jl diff --git a/debug_test/sound_wave_tests.jl b/moment_kinetics/debug_test/sound_wave_tests.jl similarity index 100% rename from debug_test/sound_wave_tests.jl rename to moment_kinetics/debug_test/sound_wave_tests.jl diff --git a/debug_test/wall_bc_inputs.jl b/moment_kinetics/debug_test/wall_bc_inputs.jl similarity index 100% rename from debug_test/wall_bc_inputs.jl rename to moment_kinetics/debug_test/wall_bc_inputs.jl diff --git a/debug_test/wall_bc_tests.jl b/moment_kinetics/debug_test/wall_bc_tests.jl similarity index 100% rename from debug_test/wall_bc_tests.jl rename to moment_kinetics/debug_test/wall_bc_tests.jl diff --git a/src/advection.jl b/moment_kinetics/src/advection.jl similarity index 100% rename from src/advection.jl rename to moment_kinetics/src/advection.jl diff --git a/src/analysis.jl b/moment_kinetics/src/analysis.jl similarity index 100% rename from src/analysis.jl rename to moment_kinetics/src/analysis.jl diff --git a/src/array_allocation.jl b/moment_kinetics/src/array_allocation.jl similarity index 100% rename from src/array_allocation.jl rename to moment_kinetics/src/array_allocation.jl diff --git a/src/bgk.jl b/moment_kinetics/src/bgk.jl similarity index 100% rename from src/bgk.jl rename to moment_kinetics/src/bgk.jl diff --git a/src/calculus.jl b/moment_kinetics/src/calculus.jl similarity index 100% rename from src/calculus.jl rename to moment_kinetics/src/calculus.jl diff --git a/src/charge_exchange.jl b/moment_kinetics/src/charge_exchange.jl similarity index 100% rename from src/charge_exchange.jl rename to moment_kinetics/src/charge_exchange.jl diff --git a/src/chebyshev.jl b/moment_kinetics/src/chebyshev.jl similarity index 100% rename from src/chebyshev.jl rename to moment_kinetics/src/chebyshev.jl diff --git a/src/clenshaw_curtis.jl b/moment_kinetics/src/clenshaw_curtis.jl similarity index 100% rename from src/clenshaw_curtis.jl rename to moment_kinetics/src/clenshaw_curtis.jl diff --git a/src/command_line_options.jl b/moment_kinetics/src/command_line_options.jl similarity index 100% rename from src/command_line_options.jl rename to moment_kinetics/src/command_line_options.jl diff --git a/src/communication.jl b/moment_kinetics/src/communication.jl similarity index 100% rename from src/communication.jl rename to moment_kinetics/src/communication.jl diff --git a/src/constants.jl b/moment_kinetics/src/constants.jl similarity index 100% rename from src/constants.jl rename to moment_kinetics/src/constants.jl diff --git a/src/continuity.jl b/moment_kinetics/src/continuity.jl similarity index 100% rename from src/continuity.jl rename to moment_kinetics/src/continuity.jl diff --git a/src/coordinates.jl b/moment_kinetics/src/coordinates.jl similarity index 100% rename from src/coordinates.jl rename to moment_kinetics/src/coordinates.jl diff --git a/src/debugging.jl b/moment_kinetics/src/debugging.jl similarity index 100% rename from src/debugging.jl rename to moment_kinetics/src/debugging.jl diff --git a/src/derivatives.jl b/moment_kinetics/src/derivatives.jl similarity index 100% rename from src/derivatives.jl rename to moment_kinetics/src/derivatives.jl diff --git a/src/em_fields.jl b/moment_kinetics/src/em_fields.jl similarity index 100% rename from src/em_fields.jl rename to moment_kinetics/src/em_fields.jl diff --git a/src/energy_equation.jl b/moment_kinetics/src/energy_equation.jl similarity index 100% rename from src/energy_equation.jl rename to moment_kinetics/src/energy_equation.jl diff --git a/src/external_sources.jl b/moment_kinetics/src/external_sources.jl similarity index 100% rename from src/external_sources.jl rename to moment_kinetics/src/external_sources.jl diff --git a/src/file_io.jl b/moment_kinetics/src/file_io.jl similarity index 99% rename from src/file_io.jl rename to moment_kinetics/src/file_io.jl index 3f7f8d9dc..d1ae1942c 100644 --- a/src/file_io.jl +++ b/moment_kinetics/src/file_io.jl @@ -291,7 +291,7 @@ function write_provenance_tracking_info!(fid, parallel_io, run_id, restart_time_ description="Total number of MPI ranks used for the run") # Get current git hash for code - project_dir = dirname(dirname(@__FILE__)) + project_dir = dirname(dirname(dirname(@__FILE__))) repo = GitRepo(project_dir) git_commit_hash = string(LibGit2.GitHash(LibGit2.peel(LibGit2.GitCommit, LibGit2.head(repo)))) if LibGit2.isdirty(repo) diff --git a/src/file_io_hdf5.jl b/moment_kinetics/src/file_io_hdf5.jl similarity index 100% rename from src/file_io_hdf5.jl rename to moment_kinetics/src/file_io_hdf5.jl diff --git a/src/file_io_netcdf.jl b/moment_kinetics/src/file_io_netcdf.jl similarity index 100% rename from src/file_io_netcdf.jl rename to moment_kinetics/src/file_io_netcdf.jl diff --git a/src/finite_differences.jl b/moment_kinetics/src/finite_differences.jl similarity index 100% rename from src/finite_differences.jl rename to moment_kinetics/src/finite_differences.jl diff --git a/src/fokker_planck.jl b/moment_kinetics/src/fokker_planck.jl similarity index 100% rename from src/fokker_planck.jl rename to moment_kinetics/src/fokker_planck.jl diff --git a/src/fokker_planck_calculus.jl b/moment_kinetics/src/fokker_planck_calculus.jl similarity index 100% rename from src/fokker_planck_calculus.jl rename to moment_kinetics/src/fokker_planck_calculus.jl diff --git a/src/fokker_planck_test.jl b/moment_kinetics/src/fokker_planck_test.jl similarity index 100% rename from src/fokker_planck_test.jl rename to moment_kinetics/src/fokker_planck_test.jl diff --git a/src/force_balance.jl b/moment_kinetics/src/force_balance.jl similarity index 100% rename from src/force_balance.jl rename to moment_kinetics/src/force_balance.jl diff --git a/src/gauss_legendre.jl b/moment_kinetics/src/gauss_legendre.jl similarity index 100% rename from src/gauss_legendre.jl rename to moment_kinetics/src/gauss_legendre.jl diff --git a/src/hermite_spline_interpolation.jl b/moment_kinetics/src/hermite_spline_interpolation.jl similarity index 100% rename from src/hermite_spline_interpolation.jl rename to moment_kinetics/src/hermite_spline_interpolation.jl diff --git a/src/initial_conditions.jl b/moment_kinetics/src/initial_conditions.jl similarity index 100% rename from src/initial_conditions.jl rename to moment_kinetics/src/initial_conditions.jl diff --git a/src/input_structs.jl b/moment_kinetics/src/input_structs.jl similarity index 100% rename from src/input_structs.jl rename to moment_kinetics/src/input_structs.jl diff --git a/src/interpolation.jl b/moment_kinetics/src/interpolation.jl similarity index 100% rename from src/interpolation.jl rename to moment_kinetics/src/interpolation.jl diff --git a/src/ionization.jl b/moment_kinetics/src/ionization.jl similarity index 100% rename from src/ionization.jl rename to moment_kinetics/src/ionization.jl diff --git a/src/krook_collisions.jl b/moment_kinetics/src/krook_collisions.jl similarity index 100% rename from src/krook_collisions.jl rename to moment_kinetics/src/krook_collisions.jl diff --git a/src/load_data.jl b/moment_kinetics/src/load_data.jl similarity index 100% rename from src/load_data.jl rename to moment_kinetics/src/load_data.jl diff --git a/src/looping.jl b/moment_kinetics/src/looping.jl similarity index 100% rename from src/looping.jl rename to moment_kinetics/src/looping.jl diff --git a/src/manufactured_solns.jl b/moment_kinetics/src/manufactured_solns.jl similarity index 100% rename from src/manufactured_solns.jl rename to moment_kinetics/src/manufactured_solns.jl diff --git a/src/moment_constraints.jl b/moment_kinetics/src/moment_constraints.jl similarity index 100% rename from src/moment_constraints.jl rename to moment_kinetics/src/moment_constraints.jl diff --git a/src/moment_kinetics.jl b/moment_kinetics/src/moment_kinetics.jl similarity index 99% rename from src/moment_kinetics.jl rename to moment_kinetics/src/moment_kinetics.jl index 32178039a..56c2cee32 100644 --- a/src/moment_kinetics.jl +++ b/moment_kinetics/src/moment_kinetics.jl @@ -9,7 +9,7 @@ using MPI # Include submodules from other source files # Note that order of includes matters - things used in one module must already # be defined -include("../machines/shared/machine_setup.jl") # Included so Documenter.jl can find its docs +include("../../machines/shared/machine_setup.jl") # Included so Documenter.jl can find its docs include("command_line_options.jl") include("constants.jl") include("debugging.jl") diff --git a/src/moment_kinetics_input.jl b/moment_kinetics/src/moment_kinetics_input.jl similarity index 100% rename from src/moment_kinetics_input.jl rename to moment_kinetics/src/moment_kinetics_input.jl diff --git a/src/moment_kinetics_structs.jl b/moment_kinetics/src/moment_kinetics_structs.jl similarity index 100% rename from src/moment_kinetics_structs.jl rename to moment_kinetics/src/moment_kinetics_structs.jl diff --git a/src/neutral_advection.jl b/moment_kinetics/src/neutral_advection.jl similarity index 100% rename from src/neutral_advection.jl rename to moment_kinetics/src/neutral_advection.jl diff --git a/src/neutral_r_advection.jl b/moment_kinetics/src/neutral_r_advection.jl similarity index 100% rename from src/neutral_r_advection.jl rename to moment_kinetics/src/neutral_r_advection.jl diff --git a/src/neutral_vz_advection.jl b/moment_kinetics/src/neutral_vz_advection.jl similarity index 100% rename from src/neutral_vz_advection.jl rename to moment_kinetics/src/neutral_vz_advection.jl diff --git a/src/neutral_z_advection.jl b/moment_kinetics/src/neutral_z_advection.jl similarity index 100% rename from src/neutral_z_advection.jl rename to moment_kinetics/src/neutral_z_advection.jl diff --git a/src/numerical_dissipation.jl b/moment_kinetics/src/numerical_dissipation.jl similarity index 100% rename from src/numerical_dissipation.jl rename to moment_kinetics/src/numerical_dissipation.jl diff --git a/src/parameter_scans.jl b/moment_kinetics/src/parameter_scans.jl similarity index 100% rename from src/parameter_scans.jl rename to moment_kinetics/src/parameter_scans.jl diff --git a/src/quadrature.jl b/moment_kinetics/src/quadrature.jl similarity index 100% rename from src/quadrature.jl rename to moment_kinetics/src/quadrature.jl diff --git a/src/r_advection.jl b/moment_kinetics/src/r_advection.jl similarity index 100% rename from src/r_advection.jl rename to moment_kinetics/src/r_advection.jl diff --git a/src/reference_parameters.jl b/moment_kinetics/src/reference_parameters.jl similarity index 100% rename from src/reference_parameters.jl rename to moment_kinetics/src/reference_parameters.jl diff --git a/src/source_terms.jl b/moment_kinetics/src/source_terms.jl similarity index 100% rename from src/source_terms.jl rename to moment_kinetics/src/source_terms.jl diff --git a/src/time_advance.jl b/moment_kinetics/src/time_advance.jl similarity index 100% rename from src/time_advance.jl rename to moment_kinetics/src/time_advance.jl diff --git a/src/type_definitions.jl b/moment_kinetics/src/type_definitions.jl similarity index 100% rename from src/type_definitions.jl rename to moment_kinetics/src/type_definitions.jl diff --git a/src/utils.jl b/moment_kinetics/src/utils.jl similarity index 100% rename from src/utils.jl rename to moment_kinetics/src/utils.jl diff --git a/src/velocity_grid_transforms.jl b/moment_kinetics/src/velocity_grid_transforms.jl similarity index 100% rename from src/velocity_grid_transforms.jl rename to moment_kinetics/src/velocity_grid_transforms.jl diff --git a/src/velocity_moments.jl b/moment_kinetics/src/velocity_moments.jl similarity index 100% rename from src/velocity_moments.jl rename to moment_kinetics/src/velocity_moments.jl diff --git a/src/vpa_advection.jl b/moment_kinetics/src/vpa_advection.jl similarity index 100% rename from src/vpa_advection.jl rename to moment_kinetics/src/vpa_advection.jl diff --git a/src/vperp_advection.jl b/moment_kinetics/src/vperp_advection.jl similarity index 100% rename from src/vperp_advection.jl rename to moment_kinetics/src/vperp_advection.jl diff --git a/src/z_advection.jl b/moment_kinetics/src/z_advection.jl similarity index 100% rename from src/z_advection.jl rename to moment_kinetics/src/z_advection.jl diff --git a/test/Krook_collisions_tests.jl b/moment_kinetics/test/Krook_collisions_tests.jl similarity index 100% rename from test/Krook_collisions_tests.jl rename to moment_kinetics/test/Krook_collisions_tests.jl diff --git a/test/Project.toml b/moment_kinetics/test/Project.toml similarity index 100% rename from test/Project.toml rename to moment_kinetics/test/Project.toml diff --git a/test/calculus_tests.jl b/moment_kinetics/test/calculus_tests.jl similarity index 100% rename from test/calculus_tests.jl rename to moment_kinetics/test/calculus_tests.jl diff --git a/test/fokker_planck_tests.jl b/moment_kinetics/test/fokker_planck_tests.jl similarity index 100% rename from test/fokker_planck_tests.jl rename to moment_kinetics/test/fokker_planck_tests.jl diff --git a/test/fokker_planck_time_evolution_tests.jl b/moment_kinetics/test/fokker_planck_time_evolution_tests.jl similarity index 100% rename from test/fokker_planck_time_evolution_tests.jl rename to moment_kinetics/test/fokker_planck_time_evolution_tests.jl diff --git a/test/harrisonthompson.jl b/moment_kinetics/test/harrisonthompson.jl similarity index 100% rename from test/harrisonthompson.jl rename to moment_kinetics/test/harrisonthompson.jl diff --git a/test/interpolation_tests.jl b/moment_kinetics/test/interpolation_tests.jl similarity index 100% rename from test/interpolation_tests.jl rename to moment_kinetics/test/interpolation_tests.jl diff --git a/test/loop_setup_tests.jl b/moment_kinetics/test/loop_setup_tests.jl similarity index 100% rename from test/loop_setup_tests.jl rename to moment_kinetics/test/loop_setup_tests.jl diff --git a/test/nonlinear_sound_wave_inputs_and_expected_data.jl b/moment_kinetics/test/nonlinear_sound_wave_inputs_and_expected_data.jl similarity index 100% rename from test/nonlinear_sound_wave_inputs_and_expected_data.jl rename to moment_kinetics/test/nonlinear_sound_wave_inputs_and_expected_data.jl diff --git a/test/nonlinear_sound_wave_tests.jl b/moment_kinetics/test/nonlinear_sound_wave_tests.jl similarity index 100% rename from test/nonlinear_sound_wave_tests.jl rename to moment_kinetics/test/nonlinear_sound_wave_tests.jl diff --git a/test/recycling_fraction_tests.jl b/moment_kinetics/test/recycling_fraction_tests.jl similarity index 100% rename from test/recycling_fraction_tests.jl rename to moment_kinetics/test/recycling_fraction_tests.jl diff --git a/test/restart_interpolation_tests.jl b/moment_kinetics/test/restart_interpolation_tests.jl similarity index 100% rename from test/restart_interpolation_tests.jl rename to moment_kinetics/test/restart_interpolation_tests.jl diff --git a/test/runtests.jl b/moment_kinetics/test/runtests.jl similarity index 100% rename from test/runtests.jl rename to moment_kinetics/test/runtests.jl diff --git a/test/setup.jl b/moment_kinetics/test/setup.jl similarity index 100% rename from test/setup.jl rename to moment_kinetics/test/setup.jl diff --git a/test/sound_wave_tests.jl b/moment_kinetics/test/sound_wave_tests.jl similarity index 100% rename from test/sound_wave_tests.jl rename to moment_kinetics/test/sound_wave_tests.jl diff --git a/test/velocity_integral_tests.jl b/moment_kinetics/test/velocity_integral_tests.jl similarity index 100% rename from test/velocity_integral_tests.jl rename to moment_kinetics/test/velocity_integral_tests.jl diff --git a/test/wall_bc_tests.jl b/moment_kinetics/test/wall_bc_tests.jl similarity index 100% rename from test/wall_bc_tests.jl rename to moment_kinetics/test/wall_bc_tests.jl From c0e94aedc3d018307f484ca7228ddb042265794f Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 19 Dec 2023 16:54:58 +0000 Subject: [PATCH 06/80] Make NCDatasets an optional dependency NetCDF file I/O is now done by an 'extension' that is available only if NCDatasets is installed. --- moment_kinetics/Project.toml | 7 +- .../{src => ext}/file_io_netcdf.jl | 56 ++++++++++++++- moment_kinetics/src/file_io.jl | 70 +++++++++++++++---- moment_kinetics/src/file_io_hdf5.jl | 2 +- moment_kinetics/src/input_structs.jl | 2 +- moment_kinetics/src/load_data.jl | 46 ++++-------- 6 files changed, 132 insertions(+), 51 deletions(-) rename moment_kinetics/{src => ext}/file_io_netcdf.jl (80%) diff --git a/moment_kinetics/Project.toml b/moment_kinetics/Project.toml index 23e5521f1..0c0e18ecc 100644 --- a/moment_kinetics/Project.toml +++ b/moment_kinetics/Project.toml @@ -20,7 +20,6 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" Measures = "442fdcdd-2543-5da2-b0f3-8c86c306513e" -NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Preferences = "21216c6a-2e73-6563-6e65-726566657250" @@ -40,6 +39,12 @@ TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" +[weakdeps] +NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" + +[extensions] +file_io_netcdf = "NCDatasets" + [compat] HDF5_jll = "<1.14, >=1.15" julia = "1.7.0" diff --git a/moment_kinetics/src/file_io_netcdf.jl b/moment_kinetics/ext/file_io_netcdf.jl similarity index 80% rename from moment_kinetics/src/file_io_netcdf.jl rename to moment_kinetics/ext/file_io_netcdf.jl index 58be77d4d..7dcd0ec81 100644 --- a/moment_kinetics/src/file_io_netcdf.jl +++ b/moment_kinetics/ext/file_io_netcdf.jl @@ -1,4 +1,18 @@ -# No separate module defined here as this file is included within the file_io module +# This extension provides an interface to the optional file_io_netcdf package, which +# provides NetCDF I/O +# +# Note that if there are errors when precompiling an extension, they may not be shown by +# default. To see the error, precompile by running +# `using Pkg; Pkg.precompile(strict=true)`. +module file_io_netcdf + +import moment_kinetics.file_io: io_has_parallel, open_output_file_implementation, + create_io_group, get_group, is_group, get_subgroup_keys, + get_variable_keys, add_attribute!, write_single_value!, + create_dynamic_variable!, append_to_dynamic_var +import moment_kinetics.load_data: open_file_to_read, load_variable, load_slice +using moment_kinetics.coordinates: coordinate +using moment_kinetics.input_structs: netcdf using NCDatasets @@ -7,7 +21,8 @@ function io_has_parallel(::Val{netcdf}) return false end -function open_output_file_netcdf(prefix, parallel_io, io_comm, mode="c") +function open_output_file_implementation(::Val{netcdf}, prefix, parallel_io, io_comm, + mode="c") parallel_io && error("NetCDF interface does not support parallel I/O") # the netcdf file will be given by output_dir/run_name with .cdf appended @@ -238,3 +253,40 @@ function append_to_dynamic_var(io_var::NCDatasets.CFVariable, return nothing end + +function open_file_to_read(::Val{netcdf}, filename) + return NCDataset(filename, "r") +end + +function load_variable(file_or_group::NCDataset, name::String) + # This overload deals with cases where fid is a NetCDF `Dataset` (which could be a + # file or a group). + try + if size(file_or_group[name].var) == () + var = file_or_group[name].var[] + else + var = copy(file_or_group[name].var) + end + if isa(var, Char) + var = (var == Char(true)) + end + return var + catch + println("An error occured while loading $name") + rethrow() + end +end + +function load_slice(file_or_group::NCDataset, name::String, slices_or_indices...) + # This overload deals with cases where fid is a NetCDF `Dataset` (which could be a + # file or a group). + try + var = file_or_group[name].var[slices_or_indices...] + return var + catch + println("An error occured while loading $name") + rethrow() + end +end + +end # file_io_netcdf diff --git a/moment_kinetics/src/file_io.jl b/moment_kinetics/src/file_io.jl index d1ae1942c..8bc271271 100644 --- a/moment_kinetics/src/file_io.jl +++ b/moment_kinetics/src/file_io.jl @@ -7,7 +7,6 @@ export get_group export open_output_file, open_ascii_output_file export setup_file_io, finish_file_io export write_data_to_ascii -export write_data_to_netcdf, write_data_to_hdf5 using ..communication using ..coordinates: coordinate @@ -25,6 +24,16 @@ using TOML @debug_shared_array using ..communication: DebugMPISharedArray +function __init__() + try + # Try to load the NCDatasets package. If the package is not installed, then + # NetCDF I/O will not be available. + Base.require(Main, :NCDatasets) + catch + # Do nothing + end +end + """ structure containing the various input/output streams """ @@ -129,6 +138,35 @@ Test if the backend supports parallel I/O. """ function io_has_parallel end +function io_has_implementation(binary_format::binary_format_type) + if binary_format == hdf5 + return true + elseif binary_format == netcdf + netcdf_ext = Base.get_extension(@__MODULE__, :file_io_netcdf) + return netcdf_ext !== nothing + else + error("Unrecognised binary format $binary_format") + end +end + +""" + check_io_implementation(binary_format) + +Check that an implementation is available for `binary_format`, raising an error if not. +""" +function check_io_implementation(binary_format) + if !io_has_implementation(binary_format) + if binary_format == netcdf + error("NCDatasets is not installed, cannot use NetCDF I/O. Re-run " + * "machines/machine-setup.sh and activate NetCDF, or install " + * "NCDatasets.") + else + error("No implementation available for binary format $binary_format") + end + end + return nothing +end + """ open the necessary output files """ @@ -886,17 +924,22 @@ Add an attribute to a file, group or variable """ function add_attribute!() end +""" +Low-level function to open a binary output file + +Each implementation (HDF5, NetCDF, etc.) defines a method of this function to open a file +of the corresponding type. +""" +function open_output_file_implementation end + """ Open an output file, selecting the backend based on io_option """ function open_output_file(prefix, binary_format, parallel_io, io_comm) - if binary_format == hdf5 - return open_output_file_hdf5(prefix, parallel_io, io_comm) - elseif binary_format == netcdf - return open_output_file_netcdf(prefix, parallel_io, io_comm) - else - error("Unsupported I/O format $binary_format") - end + check_io_implementation(binary_format) + + return open_output_file_implementation(Val(binary_format), prefix, parallel_io, + io_comm) end """ @@ -905,9 +948,13 @@ Re-open an existing output file, selecting the backend based on io_option function reopen_output_file(filename, parallel_io, io_comm) prefix, format_string = splitext(filename) if format_string == ".h5" - return open_output_file_hdf5(prefix, parallel_io, io_comm, "r+")[1] + check_io_implementation(hdf5) + return open_output_file_implementation(Val(hdf5), prefix, parallel_io, io_comm, + "r+")[1] elseif format_string == ".cdf" - return open_output_file_netcdf(prefix, parallel_io, io_comm, "a")[1] + check_io_implementation(netcdf) + return open_output_file_implementation(Val(netcdf), prefix, parallel_io, io_comm, + "a")[1] else error("Unsupported I/O format $binary_format") end @@ -1333,8 +1380,7 @@ function finish_file_io(ascii_io::Union{ascii_ios,Nothing}, return nothing end -# Include the possible implementations of binary I/O functions -include("file_io_netcdf.jl") +# Include the non-optional implementations of binary I/O functions include("file_io_hdf5.jl") """ diff --git a/moment_kinetics/src/file_io_hdf5.jl b/moment_kinetics/src/file_io_hdf5.jl index 652423ed1..0804b2ec7 100644 --- a/moment_kinetics/src/file_io_hdf5.jl +++ b/moment_kinetics/src/file_io_hdf5.jl @@ -7,7 +7,7 @@ function io_has_parallel(::Val{hdf5}) return HDF5.has_parallel() end -function open_output_file_hdf5(prefix, parallel_io, io_comm, mode="cw") +function open_output_file_implementation(::Val{hdf5}, prefix, parallel_io, io_comm, mode="cw") # the hdf5 file will be given by output_dir/run_name with .h5 appended filename = string(prefix, ".h5") # create the new HDF5 file diff --git a/moment_kinetics/src/input_structs.jl b/moment_kinetics/src/input_structs.jl index 6e51c24b3..99c4be9eb 100644 --- a/moment_kinetics/src/input_structs.jl +++ b/moment_kinetics/src/input_structs.jl @@ -341,7 +341,7 @@ mutable struct geometry_input end @enum binary_format_type hdf5 netcdf -export hdf5, netcdf +export binary_format_type, hdf5, netcdf """ Settings and input for setting up file I/O diff --git a/moment_kinetics/src/load_data.jl b/moment_kinetics/src/load_data.jl index 7a2152a76..de4e21fbe 100644 --- a/moment_kinetics/src/load_data.jl +++ b/moment_kinetics/src/load_data.jl @@ -16,15 +16,19 @@ export load_species_data using ..array_allocation: allocate_float using ..coordinates: coordinate, define_coordinate -using ..file_io: get_group, get_subgroup_keys, get_variable_keys -using ..input_structs: advection_input, grid_input +using ..file_io: check_io_implementation, get_group, get_subgroup_keys, get_variable_keys +using ..input_structs: advection_input, grid_input, hdf5, netcdf using ..interpolation: interpolate_to_grid_1d! using ..looping using ..type_definitions: mk_int using HDF5 using MPI -using NCDatasets + +function open_file_to_read end +function open_file_to_read(::Val{hdf5}, filename) + return h5open(filename, "r") +end """ """ @@ -51,13 +55,16 @@ function open_readonly_output_file(run_name, ext; iblock=0, printout=false) print("Opening ", filename, " to read HDF5 data...") end # open the HDF5 file with given filename for reading - fid = h5open(filename, "r") + check_io_implementation(hdf5) + fid = open_file_to_read(Val(hdf5), filename) else if printout print("Opening ", filename, " to read NetCDF data...") end + # open the netcdf file with given filename for reading - fid = NCDataset(filename, "r") + check_io_implementation(netcdf) + fid = open_file_to_read(Val(netcdf), filename) end if printout println("done.") @@ -94,24 +101,6 @@ function load_variable(file_or_group::HDF5.H5DataStore, name::String) rethrow() end end -function load_variable(file_or_group::NCDataset, name::String) - # This overload deals with cases where fid is a NetCDF `Dataset` (which could be a - # file or a group). - try - if size(file_or_group[name].var) == () - var = file_or_group[name].var[] - else - var = copy(file_or_group[name].var) - end - if isa(var, Char) - var = (var == Char(true)) - end - return var - catch - println("An error occured while loading $name") - rethrow() - end -end """ Load a slice of a single variable from a file @@ -127,17 +116,6 @@ function load_slice(file_or_group::HDF5.H5DataStore, name::String, slices_or_ind rethrow() end end -function load_slice(file_or_group::NCDataset, name::String, slices_or_indices...) - # This overload deals with cases where fid is a NetCDF `Dataset` (which could be a - # file or a group). - try - var = file_or_group[name].var[slices_or_indices...] - return var - catch - println("An error occured while loading $name") - rethrow() - end -end """ read_Dict_from_section(file_or_group, section_name; ignore_subsections=false) From 36568dcbb4da5ade12f46b5eccc16252f513ab0d Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 19 Dec 2023 17:00:16 +0000 Subject: [PATCH 07/80] Remove brackets from generic function definitions The 'generic' definitions are intended only to define a name that can be imported by each implementation to add an implementation-specific method. If they are defined with brackets after the name (as several were previously) then there is a zero-argument method of the function that does nothing, which might sometimes be confusing. Therefore remove those brackets. --- moment_kinetics/src/file_io.jl | 16 ++++++++-------- moment_kinetics/src/load_data.jl | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/moment_kinetics/src/file_io.jl b/moment_kinetics/src/file_io.jl index 8bc271271..12f8891e8 100644 --- a/moment_kinetics/src/file_io.jl +++ b/moment_kinetics/src/file_io.jl @@ -217,22 +217,22 @@ end """ Get a (sub-)group from a file or group """ -function get_group() end +function get_group end """ Test if a member of a (sub-)group is a group """ -function is_group() end +function is_group end """ Get names of all subgroups """ -function get_subgroup_keys() end +function get_subgroup_keys end """ Get names of all variables """ -function get_variable_keys() end +function get_variable_keys end """ write_single_value!(file_or_group, name, value; description=nothing, units=nothing) @@ -240,7 +240,7 @@ function get_variable_keys() end Write a single variable to a file or group. If a description or units are passed, add as attributes of the variable. """ -function write_single_value!() end +function write_single_value! end """ write some overview information for the simulation to the binary file @@ -613,7 +613,7 @@ array dimensions. The species dimension does not have a `coordinate`, so the num species is passed as `nspecies`. A description and/or units can be added with the keyword arguments. """ -function create_dynamic_variable!() end +function create_dynamic_variable! end """ define dynamic (time-evolving) moment variables for writing to the hdf5 file @@ -922,7 +922,7 @@ end """ Add an attribute to a file, group or variable """ -function add_attribute!() end +function add_attribute! end """ Low-level function to open a binary output file @@ -1163,7 +1163,7 @@ used to get the ranges to write from/to (needed for parallel I/O) - the entries `coords` tuple can be either `coordinate` instances or integers (for an integer `n` the range is `1:n`). """ -function append_to_dynamic_var() end +function append_to_dynamic_var end @debug_shared_array begin function append_to_dynamic_var(data::DebugMPISharedArray, args...; kwargs...) diff --git a/moment_kinetics/src/load_data.jl b/moment_kinetics/src/load_data.jl index de4e21fbe..567d4949b 100644 --- a/moment_kinetics/src/load_data.jl +++ b/moment_kinetics/src/load_data.jl @@ -90,7 +90,7 @@ end """ Load a single variable from a file """ -function load_variable() end +function load_variable end function load_variable(file_or_group::HDF5.H5DataStore, name::String) # This overload deals with cases where fid is an HDF5 `File` or `Group` (`H5DataStore` # is the abstract super-type for both @@ -105,7 +105,7 @@ end """ Load a slice of a single variable from a file """ -function load_slice() end +function load_slice end function load_slice(file_or_group::HDF5.H5DataStore, name::String, slices_or_indices...) # This overload deals with cases where fid is an HDF5 `File` or `Group` (`H5DataStore` # is the abstract super-type for both From bdbf1b8b8f381aab81179c12dc8714fef6b92cf6 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 19 Dec 2023 17:23:05 +0000 Subject: [PATCH 08/80] Remove TimerOutputs from 'precompile runs' The TimerOutput argument is not needed any more, so do not need the TimerOutputs dependency. --- util/precompile_run.jl | 1 - util/precompile_run_long.jl | 1 - util/precompile_run_long_debug1.jl | 1 - util/precompile_run_short.jl | 1 - 4 files changed, 4 deletions(-) diff --git a/util/precompile_run.jl b/util/precompile_run.jl index 74bacf22c..bd77578b9 100644 --- a/util/precompile_run.jl +++ b/util/precompile_run.jl @@ -2,7 +2,6 @@ using Pkg Pkg.activate(".") -using TimerOutputs using moment_kinetics # Create a temporary directory for test output diff --git a/util/precompile_run_long.jl b/util/precompile_run_long.jl index f1da79fd2..e2eee8e8e 100644 --- a/util/precompile_run_long.jl +++ b/util/precompile_run_long.jl @@ -5,7 +5,6 @@ push!(ARGS, "--long") using Pkg Pkg.activate(".") -using TimerOutputs using moment_kinetics # Create a temporary directory for test output diff --git a/util/precompile_run_long_debug1.jl b/util/precompile_run_long_debug1.jl index f53349c9e..d5dfab3b7 100644 --- a/util/precompile_run_long_debug1.jl +++ b/util/precompile_run_long_debug1.jl @@ -5,7 +5,6 @@ push!(ARGS, "--long", "--debug", "1") using Pkg Pkg.activate(".") -using TimerOutputs using moment_kinetics # Create a temporary directory for test output diff --git a/util/precompile_run_short.jl b/util/precompile_run_short.jl index ffac944a0..b0e37c86f 100644 --- a/util/precompile_run_short.jl +++ b/util/precompile_run_short.jl @@ -2,7 +2,6 @@ using Pkg Pkg.activate(".") -using TimerOutputs using moment_kinetics # Create a temporary directory for test output From 093bdf69cf9cbcef84c3ae081f3d10c3ae376c1b Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 19 Dec 2023 19:49:49 +0000 Subject: [PATCH 09/80] Make Symbolics an optional dependency, by moving MMS to an extension --- moment_kinetics/Project.toml | 4 +- moment_kinetics/ext/manufactured_solns_ext.jl | 576 ++++++++++++++++ moment_kinetics/src/manufactured_solns.jl | 617 ++---------------- moment_kinetics/src/moment_kinetics_input.jl | 10 + 4 files changed, 634 insertions(+), 573 deletions(-) create mode 100644 moment_kinetics/ext/manufactured_solns_ext.jl diff --git a/moment_kinetics/Project.toml b/moment_kinetics/Project.toml index 0c0e18ecc..2b4d2f3fb 100644 --- a/moment_kinetics/Project.toml +++ b/moment_kinetics/Project.toml @@ -12,7 +12,6 @@ FastGaussQuadrature = "442a2c76-b920-505d-bb47-c5924d526838" Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" HDF5_jll = "0234f1f7-429e-5d53-9886-15a909be8d59" -IfElse = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173" Interpolations = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59" LegendrePolynomials = "3db4a2ba-fc88-11e8-3e01-49c72059a882" LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" @@ -33,7 +32,6 @@ SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" SuiteSparse = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" -Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" @@ -41,9 +39,11 @@ Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" [weakdeps] NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" +Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7" [extensions] file_io_netcdf = "NCDatasets" +manufactured_solns_ext = "Symbolics" [compat] HDF5_jll = "<1.14, >=1.15" diff --git a/moment_kinetics/ext/manufactured_solns_ext.jl b/moment_kinetics/ext/manufactured_solns_ext.jl new file mode 100644 index 000000000..6e9318deb --- /dev/null +++ b/moment_kinetics/ext/manufactured_solns_ext.jl @@ -0,0 +1,576 @@ +# This extension provides the actual implementation for the manufactured_solns module, +# which is available only if the Symbolics package is installed +# +# Note that if there are errors when precompiling an extension, they may not be shown by +# default. To see the error, precompile by running +# `using Pkg; Pkg.precompile(strict=true)`. +module manufactured_solns_ext + +using moment_kinetics.input_structs +using moment_kinetics.looping +using moment_kinetics.type_definitions: mk_int + +import moment_kinetics.manufactured_solns: manufactured_solutions, manufactured_sources_setup, + manufactured_electric_fields + +using Symbolics +using IfElse + + @variables r z vpa vperp t vz vr vzeta + typed_zero(vz) = zero(vz) + @register_symbolic typed_zero(vz) + zero_val = 1.0e-8 + #epsilon_offset = 0.001 + + dfni_vpa_power_opt = "4" #"2" + if dfni_vpa_power_opt == "2" + pvpa = 2 + nconst = 0.25 + pconst = 3.0/4.0 + fluxconst = 0.5 + elseif dfni_vpa_power_opt == "4" + pvpa = 4 + nconst = 3.0/8.0 + pconst = 15.0/8.0 + fluxconst = 1.0 + end + + #standard functions for building densities + function nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha) + if r_bc == "periodic" + nplus = exp(sqrt(epsilon + 0.5 - z/Lz)) * exp(1.0 + 0.05*sin(2.0*pi*r/Lr)*((1.0 - alpha)*cos(pi*z/Lz) + alpha)) + elseif r_bc == "Dirichlet" + nplus = exp(1.0 - 0.2*r/Lr) + end + return nplus + end + + function nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha) + if r_bc == "periodic" + nminus = exp(sqrt(epsilon + 0.5 + z/Lz)) * exp(1.0 + 0.05*sin(2.0*pi*r/Lr)*((1.0 - alpha)*cos(pi*z/Lz) + alpha)) + elseif r_bc == "Dirichlet" + nminus = exp(1.0 - 0.2*r/Lr) + end + return nminus + end + + function nzero_sym(Lr,Lz,r_bc,z_bc,alpha) + if r_bc == "periodic" + nzero = exp(1.0 + 0.05*sin(2.0*pi*r/Lr)*((1.0 - alpha)*cos(pi*z/Lz) + alpha)) + elseif r_bc == "Dirichlet" + nzero = exp(1.0 - 0.2*r/Lr) + end + return nzero + end + + function knudsen_cosine(composition) + T_wall = composition.T_wall + exponetial = exp( - (vz^2 + vr^2 + vzeta^2)/T_wall ) + if composition.use_test_neutral_wall_pdf + #test dfn + knudsen_pdf = (4.0/T_wall^(5.0/2.0))*abs(vz)*exponetial + else + #proper Knudsen dfn + # prefac here may cause problems with NaNs if vz = vr = vzeta = 0 is on grid + fac = abs(vz)/sqrt(vz^2 + vr^2 + vzeta^2) + prefac = IfElse.ifelse( abs(vz) < 1000.0*zero_val,typed_zero(vz),fac) + knudsen_pdf = (3.0*sqrt(pi)/T_wall^2)*prefac*exponetial + end + return knudsen_pdf + end + + # neutral density symbolic function + function densn_sym(Lr, Lz, r_bc, z_bc, geometry, composition, + manufactured_solns_input, species) + if manufactured_solns_input.type == "default" + if z_bc == "periodic" + if r_bc == "periodic" + densn = 1.5 + 0.1*(cos(2.0*pi*r/Lr) + cos(2.0*pi*z/Lz)) #*sin(2.0*pi*t) + elseif r_bc == "Dirichlet" + densn = 1.5 + 0.3*r/Lr + end + elseif z_bc == "wall" + T_wall = composition.T_wall + Bzed = geometry.Bzed + Bmag = geometry.Bmag + epsilon = manufactured_solns_input.epsilon_offset + alpha = manufactured_solns_input.alpha_switch + Gamma_minus = fluxconst*(Bzed/Bmag)*nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)/sqrt(pi) + Gamma_plus = fluxconst*(Bzed/Bmag)*nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)/sqrt(pi) + # exact integral of corresponding dfnn below + if composition.use_test_neutral_wall_pdf + #test + prefactor = 2.0/sqrt(pi*T_wall) + else + #proper prefactor + prefactor = 3.0*sqrt(pi)/(4.0*sqrt(T_wall)) + end + densn = prefactor*(Gamma_minus*(0.5 - z/Lz)^2 + Gamma_plus*(0.5 + z/Lz)^2 + 2.0 ) + else + densn = 1.0 + end + elseif manufactured_solns_input.type == "2D-instability" + densn = 1.5 + 0.1*(cos(2.0*pi*r/Lr) + cos(2.0*pi*z/Lz)) + else + error("Unrecognized option " + * "manufactured_solns:type=$(manufactured_solns_input.type)") + end + return densn + end + + # neutral distribution symbolic function + function dfnn_sym(Lr, Lz, r_bc, z_bc, geometry, composition, manufactured_solns_input, + species) + densn = densn_sym(Lr, Lz, r_bc, z_bc, geometry, composition, + manufactured_solns_input, species) + if z_bc == "periodic" + dfnn = densn * exp( - vz^2 - vr^2 - vzeta^2) + elseif z_bc == "wall" + Hplus = 0.5*(sign(vz) + 1.0) + Hminus = 0.5*(sign(-vz) + 1.0) + FKw = knudsen_cosine(composition) + Bzed = geometry.Bzed + Bmag = geometry.Bmag + epsilon = manufactured_solns_input.epsilon_offset + alpha = manufactured_solns_input.alpha_switch + Gamma_minus = fluxconst*(Bzed/Bmag)*nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)/sqrt(pi) + Gamma_plus = fluxconst*(Bzed/Bmag)*nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)/sqrt(pi) + dfnn = Hplus *( Gamma_minus*( 0.5 - z/Lz)^2 + 1.0 )*FKw + Hminus*( Gamma_plus*( 0.5 + z/Lz)^2 + 1.0 )*FKw + end + return dfnn + end + function gyroaveraged_dfnn_sym(Lr, Lz, r_bc, z_bc, geometry, composition, + manufactured_solns_input, species) + densn = densn_sym(Lr, Lz, r_bc, z_bc, geometry, composition, + manufactured_solns_input, species) + #if (r_bc == "periodic" && z_bc == "periodic") + dfnn = densn * exp( - vpa^2 - vperp^2 ) + #end + return dfnn + end + + # ion density symbolic function + function densi_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, species) + if manufactured_solns_input.type == "default" + if z_bc == "periodic" + if r_bc == "periodic" + densi = 1.5 + 0.1*(sin(2.0*pi*r/Lr) + sin(2.0*pi*z/Lz))#*sin(2.0*pi*t) + elseif r_bc == "Dirichlet" + #densi = 1.0 + 0.5*sin(2.0*pi*z/Lz)*(r/Lr + 0.5) + 0.2*sin(2.0*pi*r/Lr)*sin(2.0*pi*t) + #densi = 1.0 + 0.5*sin(2.0*pi*z/Lz)*(r/Lr + 0.5) + sin(2.0*pi*r/Lr)*sin(2.0*pi*t) + densi = 1.0 + 0.5*(r/Lr)*sin(2.0*pi*z/Lz) + end + elseif z_bc == "wall" + epsilon = manufactured_solns_input.epsilon_offset + alpha = manufactured_solns_input.alpha_switch + densi = nconst*(0.5 - z/Lz)*nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha) + nconst*(z/Lz + 0.5)*nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha) + (z/Lz + 0.5)*(0.5 - z/Lz)*nzero_sym(Lr,Lz,r_bc,z_bc,alpha) #+ 0.5*(r/Lr + 0.5) + 0.5*(z/Lz + 0.5) + end + elseif manufactured_solns_input.type == "2D-instability" + # Input for instability test + background_wavenumber = 1 + round(mk_int, + species.z_IC.temperature_phase) + initial_density = species.initial_density + density_amplitude = species.z_IC.density_amplitude + density_phase = species.z_IC.density_phase + T0 = Ti_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, + species) + eta0 = (initial_density + * (1.0 + density_amplitude + * sin(2.0*π*background_wavenumber*z/Lz + + density_phase))) + densi = eta0^((T0/(1+T0))) + else + error("Unrecognized option " + * "manufactured_solns:type=$(manufactured_solns_input.type)") + end + return densi + end + + function Ti_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, species) + if manufactured_solns_input.type == "default" + error("Ti_sym() is not used for the default case") + elseif manufactured_solns_input.type == "2D-instability" + background_wavenumber = 1 + round(mk_int, + species.z_IC.temperature_phase) + initial_temperature = species.initial_temperature + temperature_amplitude = species.z_IC.temperature_amplitude + T0 = (initial_temperature + * (1.0 + temperature_amplitude + * sin(2.0*π*background_wavenumber*z/Lz) + )) + else + error("Unrecognized option " + * "manufactured_solns:type=$(manufactured_solns_input.type)") + end + end + + # ion mean parallel flow symbolic function + function upari_sym(Lr,Lz,r_bc,z_bc,composition,geometry,nr,manufactured_solns_input,species) + if z_bc == "periodic" + upari = 0.0 + elseif z_bc == "wall" + densi = densi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species) + Er, Ez, phi = electric_fields(Lr,Lz,r_bc,z_bc,composition,nr,manufactured_solns_input,species) + rhostar = geometry.rhostar + bzed = geometry.bzed + epsilon = manufactured_solns_input.epsilon_offset + alpha = manufactured_solns_input.alpha_switch + upari = ( (fluxconst/(sqrt(pi)*densi))*((z/Lz + 0.5)*nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha) + - (0.5 - z/Lz)*nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)) + + alpha*(rhostar/(2.0*bzed))*Er ) + end + return upari + end + + # ion parallel pressure symbolic function + function ppari_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species) + # normalisation factor due to pressure normalisation convention in master pref = nref mref cref^2 + norm_fac = 0.5 + if z_bc == "periodic" + densi = densi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species) + ppari = densi + elseif z_bc == "wall" + densi = densi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species) + epsilon = manufactured_solns_input.epsilon_offset + alpha = manufactured_solns_input.alpha_switch + ppari = ( pconst*((0.5 - z/Lz)*nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha) + + (z/Lz + 0.5)*nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)) + + (z/Lz + 0.5)*(0.5 - z/Lz)*nzero_sym(Lr,Lz,r_bc,z_bc,alpha) + - (2.0/(pi*densi))*((z/Lz + 0.5)*nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha) + - (0.5 - z/Lz)*nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha))^2 ) + end + return ppari*norm_fac + end + + # ion perpendicular pressure symbolic function + function pperpi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species,nvperp) + densi = densi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species) + normfac = 0.5 # if pressure normalised to 0.5* nref * Tref = mref cref^2 + #normfac = 1.0 # if pressure normalised to nref*Tref + if nvperp > 1 + pperpi = densi # simple vperp^2 dependence of dfni + else + pperpi = 0.0 # marginalised model has nvperp = 1, vperp[1] = 0 + end + return pperpi*normfac + end + + # ion thermal speed symbolic function + function vthi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species,nvperp) + densi = densi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species) + ppari = ppari_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species) + pperpi = pperpi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species,nvperp) + isotropic_pressure = (1.0/3.0)*(ppari + 2.0*pperpi) + normfac = 2.0 # if pressure normalised to 0.5* nref * Tref = mref cref^2 + #normfac = 1.0 # if pressure normalised to nref*Tref + if nvperp > 1 + vthi = sqrt(normfac*isotropic_pressure/densi) # thermal speed definition of 2V model + else + vthi = sqrt(normfac*ppari/densi) # thermal speed definition of 1V model + end + return vthi + end + + function jpari_into_LHS_wall_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input) + if z_bc == "periodic" + jpari_into_LHS_wall_sym = 0.0 + elseif z_bc == "wall" + #appropriate for wall bc test when Er = 0 (nr == 1) + epsilon = manufactured_solns_input.epsilon_offset + alpha = manufactured_solns_input.alpha_switch + jpari_into_LHS_wall_sym = -fluxconst*nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)/sqrt(pi) + end + return jpari_into_LHS_wall_sym + end + + # ion distribution symbolic function + function dfni_sym(Lr, Lz, r_bc, z_bc, composition, geometry, nr, + manufactured_solns_input, species) + densi = densi_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, + species) + + if manufactured_solns_input.type == "default" + # calculate the electric fields and the potential + Er, Ez, phi = electric_fields(Lr, Lz, r_bc, z_bc, composition, nr, + manufactured_solns_input, species) + + # get geometric/composition data + Bzed = geometry.Bzed + Bmag = geometry.Bmag + rhostar = geometry.rhostar + epsilon = manufactured_solns_input.epsilon_offset + alpha = manufactured_solns_input.alpha_switch + if z_bc == "periodic" + dfni = densi * exp( - vpa^2 - vperp^2) + elseif z_bc == "wall" + vpabar = vpa - alpha*(rhostar/2.0)*(Bmag/Bzed)*Er # for alpha = 1.0, effective velocity in z direction * (Bmag/Bzed) + Hplus = 0.5*(sign(vpabar) + 1.0) + Hminus = 0.5*(sign(-vpabar) + 1.0) + ffa = exp(- vperp^2) + dfni = ffa * ( nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)* (0.5 - z/Lz) * Hminus * vpabar^pvpa + nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)*(z/Lz + 0.5) * Hplus * vpabar^pvpa + nzero_sym(Lr,Lz,r_bc,z_bc,alpha)*(z/Lz + 0.5)*(0.5 - z/Lz) ) * exp( - vpabar^2 ) + end + elseif manufactured_solns_input.type == "2D-instability" + # Input for instability test + T0 = Ti_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, + species) + vth = sqrt(T0) + + # Note this is for a '1V' test + dfni = densi/vth * exp(-(vpa/vth)^2) + else + error("Unrecognized option " + * "manufactured_solns:type=$(manufactured_solns_input.type)") + end + return dfni + end + function cartesian_dfni_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, + species) + densi = densi_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, + species) + #if (r_bc == "periodic" && z_bc == "periodic") || (r_bc == "Dirichlet" && z_bc == "periodic") + dfni = densi * exp( - vz^2 - vr^2 - vzeta^2) + #end + return dfni + end + + function electric_fields(Lr, Lz, r_bc, z_bc, composition, nr, + manufactured_solns_input, species) + + # define derivative operators + Dr = Differential(r) + Dz = Differential(z) + + # get N_e factor for boltzmann response + if composition.electron_physics == boltzmann_electron_response_with_simple_sheath && nr == 1 + # so 1D MMS test with 3V neutrals where ion current can be calculated prior to knowing Er + jpari_into_LHS_wall = jpari_into_LHS_wall_sym(Lr, Lz, r_bc, z_bc, + manufactured_solns_input) + N_e = -2.0*sqrt(pi*composition.me_over_mi)*exp(-composition.phi_wall/composition.T_e)*jpari_into_LHS_wall + elseif composition.electron_physics == boltzmann_electron_response_with_simple_sheath && nr > 1 + println("ERROR: simple sheath MMS test not supported for nr > 1") + println("INFO: In general, not possible to analytically calculate jpari for sheath prior to knowing Er, but Er depends on jpari") + println("Setting N_e = 1.0. Expect MMS test to fail!") + N_e = 1.0 + elseif composition.electron_physics == boltzmann_electron_response + # all other cases + # N_e equal to reference density + N_e = 1.0 + end + + if nr > 1 # keep radial electric field + rfac = 1.0 + else # drop radial electric field + rfac = 0.0 + end + + densi = densi_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, + species) + # calculate the electric fields + dense = densi # get the electron density via quasineutrality with Zi = 1 + phi = composition.T_e*log(dense/N_e) # use the adiabatic response of electrons for me/mi -> 0 + Er = -Dr(phi)*rfac + composition.Er_constant + Ez = -Dz(phi) + + Er_expanded = expand_derivatives(Er) + Ez_expanded = expand_derivatives(Ez) + + return Er_expanded, Ez_expanded, phi + end + + function manufactured_solutions(manufactured_solns_input, Lr, Lz, r_bc, z_bc, + geometry, composition, species, nr, nvperp) + charged_species = species.charged[1] + if composition.n_neutral_species > 0 + neutral_species = species.neutral[1] + else + neutral_species = nothing + end + + densi = densi_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, + charged_species) + upari = upari_sym(Lr, Lz, r_bc, z_bc, composition, geometry, nr, manufactured_solns_input, + charged_species) + ppari = ppari_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, + charged_species) + pperpi = pperpi_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, + charged_species, nvperp) + vthi = vthi_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, + charged_species, nvperp) + dfni = dfni_sym(Lr, Lz, r_bc, z_bc, composition, geometry, nr, + manufactured_solns_input, charged_species) + + densn = densn_sym(Lr, Lz, r_bc, z_bc, geometry,composition, + manufactured_solns_input, neutral_species) + dfnn = dfnn_sym(Lr, Lz, r_bc, z_bc, geometry, composition, + manufactured_solns_input, neutral_species) + + #build julia functions from these symbolic expressions + # cf. https://docs.juliahub.com/Symbolics/eABRO/3.4.0/tutorials/symbolic_functions/ + densi_func = build_function(densi, z, r, t, expression=Val{false}) + upari_func = build_function(upari, z, r, t, expression=Val{false}) + ppari_func = build_function(ppari, z, r, t, expression=Val{false}) + pperpi_func = build_function(pperpi, z, r, t, expression=Val{false}) + vthi_func = build_function(vthi, z, r, t, expression=Val{false}) + densn_func = build_function(densn, z, r, t, expression=Val{false}) + dfni_func = build_function(dfni, vpa, vperp, z, r, t, expression=Val{false}) + dfnn_func = build_function(dfnn, vz, vr, vzeta, z, r, t, expression=Val{false}) + # return function + # call like: + # densi_func(zval, rval, tval) + # dfni_func(vpaval, vperpval, zval, rval, tval) + # densn_func(zval, rval, tval) + # dfnn_func(vzval, vrval, vzetapval, zval, rval, tval) + + manufactured_solns_list = (densi_func = densi_func, densn_func = densn_func, + dfni_func = dfni_func, dfnn_func = dfnn_func, + upari_func = upari_func, ppari_func = ppari_func, + pperpi_func = pperpi_func, vthi_func = vthi_func) + + return manufactured_solns_list + end + + function manufactured_electric_fields(Lr, Lz, r_bc, z_bc, composition, nr, + manufactured_solns_input, species) + + # calculate the electric fields and the potential + Er, Ez, phi = electric_fields(Lr, Lz, r_bc, z_bc, composition, nr, + manufactured_solns_input, species) + + Er_func = build_function(Er, z, r, t, expression=Val{false}) + Ez_func = build_function(Ez, z, r, t, expression=Val{false}) + phi_func = build_function(phi, z, r, t, expression=Val{false}) + + manufactured_E_fields = (Er_func = Er_func, Ez_func = Ez_func, phi_func = phi_func) + + return manufactured_E_fields + end + + function manufactured_sources_setup(manufactured_solns_input, r_coord, z_coord, vperp_coord, + vpa_coord, vzeta_coord, vr_coord, vz_coord, composition, geometry, collisions, + num_diss_params, species) + + charged_species = species.charged[1] + if composition.n_neutral_species > 0 + neutral_species = species.neutral[1] + else + neutral_species = nothing + end + + # ion manufactured solutions + densi = densi_sym(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, composition, + manufactured_solns_input, charged_species) + upari = upari_sym(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, composition, geometry, r_coord.n, manufactured_solns_input, charged_species) + vthi = vthi_sym(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, composition, manufactured_solns_input, + charged_species, vperp_coord.n) + dfni = dfni_sym(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, composition, + geometry, r_coord.n, manufactured_solns_input, charged_species) + #dfni in vr vz vzeta coordinates + vrvzvzeta_dfni = cartesian_dfni_sym(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, + composition, manufactured_solns_input, + charged_species) + + # neutral manufactured solutions + densn = densn_sym(r_coord.L,z_coord.L, r_coord.bc, z_coord.bc, geometry, + composition, manufactured_solns_input, neutral_species) + dfnn = dfnn_sym(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, geometry, + composition, manufactured_solns_input, neutral_species) + # gyroaverage < dfnn > in vpa vperp coordinates + gav_dfnn = gyroaveraged_dfnn_sym(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, + geometry, composition, manufactured_solns_input, + neutral_species) + + dense = densi # get the electron density via quasineutrality with Zi = 1 + + # define derivative operators + Dr = Differential(r) + Dz = Differential(z) + Dvpa = Differential(vpa) + Dvperp = Differential(vperp) + Dvz = Differential(vz) + Dt = Differential(t) + + # get geometric/composition data + Bzed = geometry.Bzed + Bmag = geometry.Bmag + rhostar = geometry.rhostar + #exceptions for cases with missing terms + if composition.n_neutral_species > 0 + cx_frequency = collisions.charge_exchange + ionization_frequency = collisions.ionization + else + cx_frequency = 0.0 + ionization_frequency = 0.0 + end + if r_coord.n > 1 # keep radial derivatives + rfac = 1.0 + else # drop radial derivative terms + rfac = 0.0 + end + + # calculate the electric fields and the potential + Er, Ez, phi = electric_fields(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, + composition, r_coord.n, manufactured_solns_input, + charged_species) + + # the ion source to maintain the manufactured solution + Si = ( Dt(dfni) + ( vpa * (Bzed/Bmag) - 0.5*rhostar*Er ) * Dz(dfni) + ( 0.5*rhostar*Ez*rfac ) * Dr(dfni) + ( 0.5*Ez*Bzed/Bmag ) * Dvpa(dfni) + + cx_frequency*( densn*dfni - densi*gav_dfnn ) - ionization_frequency*dense*gav_dfnn) + nu_krook = collisions.krook_collision_frequency_prefactor + if nu_krook > 0.0 + Ti_over_Tref = vthi^2 + if collisions.krook_collisions_option == "manual" + nuii_krook = nu_krook + else # default option + nuii_krook = nu_krook * densi * Ti_over_Tref^(-1.5) + end + if vperp_coord.n > 1 + pvth = 3 + else + pvth = 1 + end + FMaxwellian = (densi/vthi^pvth)*exp( -( ( vpa-upari)^2 + vperp^2 )/vthi^2) + Si += - nuii_krook*(FMaxwellian - dfni) + end + include_num_diss_in_MMS = true + if num_diss_params.vpa_dissipation_coefficient > 0.0 && include_num_diss_in_MMS + Si += - num_diss_params.vpa_dissipation_coefficient*Dvpa(Dvpa(dfni)) + end + if num_diss_params.vperp_dissipation_coefficient > 0.0 && include_num_diss_in_MMS + Si += - num_diss_params.vperp_dissipation_coefficient*Dvperp(Dvperp(dfni)) + end + if num_diss_params.r_dissipation_coefficient > 0.0 && include_num_diss_in_MMS + Si += - rfac*num_diss_params.r_dissipation_coefficient*Dr(Dr(dfni)) + end + if num_diss_params.z_dissipation_coefficient > 0.0 && include_num_diss_in_MMS + Si += - num_diss_params.z_dissipation_coefficient*Dz(Dz(dfni)) + end + + Source_i = expand_derivatives(Si) + + # the neutral source to maintain the manufactured solution + Sn = Dt(dfnn) + vz * Dz(dfnn) + rfac*vr * Dr(dfnn) + cx_frequency* (densi*dfnn - densn*vrvzvzeta_dfni) + ionization_frequency*dense*dfnn + if num_diss_params.vz_dissipation_coefficient > 0.0 && include_num_diss_in_MMS + Sn += - num_diss_params.vz_dissipation_coefficient*Dvz(Dvz(dfnn)) + end + if num_diss_params.r_dissipation_coefficient > 0.0 && include_num_diss_in_MMS + Sn += - rfac*num_diss_params.r_dissipation_coefficient*Dr(Dr(dfnn)) + end + if num_diss_params.z_dissipation_coefficient > 0.0 && include_num_diss_in_MMS + Sn += - num_diss_params.z_dissipation_coefficient*Dz(Dz(dfnn)) + end + + Source_n = expand_derivatives(Sn) + + Source_i_func = build_function(Source_i, vpa, vperp, z, r, t, expression=Val{false}) + Source_n_func = build_function(Source_n, vz, vr, vzeta, z, r, t, expression=Val{false}) + + if expand_derivatives(Dt(Source_i)) == 0 && expand_derivatives(Dt(Source_n)) == 0 + time_independent_sources = true + else + time_independent_sources = false + end + + return time_independent_sources, Source_i_func, Source_n_func + end + +end diff --git a/moment_kinetics/src/manufactured_solns.jl b/moment_kinetics/src/manufactured_solns.jl index 3130bd674..fb63ac598 100644 --- a/moment_kinetics/src/manufactured_solns.jl +++ b/moment_kinetics/src/manufactured_solns.jl @@ -7,594 +7,69 @@ export manufactured_sources export manufactured_electric_fields using ..array_allocation: allocate_shared_float -using ..input_structs using ..looping using ..type_definitions: mk_float, mk_int -using Symbolics -using IfElse +function manufactured_solutions end +function manufactured_sources_setup end +function manufactured_electric_fields end - @variables r z vpa vperp t vz vr vzeta - typed_zero(vz) = zero(vz) - @register_symbolic typed_zero(vz) - zero_val = 1.0e-8 - #epsilon_offset = 0.001 - - dfni_vpa_power_opt = "4" #"2" - if dfni_vpa_power_opt == "2" - pvpa = 2 - nconst = 0.25 - pconst = 3.0/4.0 - fluxconst = 0.5 - elseif dfni_vpa_power_opt == "4" - pvpa = 4 - nconst = 3.0/8.0 - pconst = 15.0/8.0 - fluxconst = 1.0 - end - - #standard functions for building densities - function nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha) - if r_bc == "periodic" - nplus = exp(sqrt(epsilon + 0.5 - z/Lz)) * exp(1.0 + 0.05*sin(2.0*pi*r/Lr)*((1.0 - alpha)*cos(pi*z/Lz) + alpha)) - elseif r_bc == "Dirichlet" - nplus = exp(1.0 - 0.2*r/Lr) - end - return nplus - end - - function nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha) - if r_bc == "periodic" - nminus = exp(sqrt(epsilon + 0.5 + z/Lz)) * exp(1.0 + 0.05*sin(2.0*pi*r/Lr)*((1.0 - alpha)*cos(pi*z/Lz) + alpha)) - elseif r_bc == "Dirichlet" - nminus = exp(1.0 - 0.2*r/Lr) - end - return nminus - end - - function nzero_sym(Lr,Lz,r_bc,z_bc,alpha) - if r_bc == "periodic" - nzero = exp(1.0 + 0.05*sin(2.0*pi*r/Lr)*((1.0 - alpha)*cos(pi*z/Lz) + alpha)) - elseif r_bc == "Dirichlet" - nzero = exp(1.0 - 0.2*r/Lr) - end - return nzero +function __init__() + try + # Try to load the Symbolics package so we can use it for manufactured solutions. + # If the package is not installed, then manufactured solutions will not be + # available. + Base.require(Main, :Symbolics) + catch + # Do nothing end +end - function knudsen_cosine(composition) - T_wall = composition.T_wall - exponetial = exp( - (vz^2 + vr^2 + vzeta^2)/T_wall ) - if composition.use_test_neutral_wall_pdf - #test dfn - knudsen_pdf = (4.0/T_wall^(5.0/2.0))*abs(vz)*exponetial - else - #proper Knudsen dfn - # prefac here may cause problems with NaNs if vz = vr = vzeta = 0 is on grid - fac = abs(vz)/sqrt(vz^2 + vr^2 + vzeta^2) - prefac = IfElse.ifelse( abs(vz) < 1000.0*zero_val,typed_zero(vz),fac) - knudsen_pdf = (3.0*sqrt(pi)/T_wall^2)*prefac*exponetial - end - return knudsen_pdf - end +# This function is defined here rather than in the extension, because the looping macros +# break if used outside the moment_kinetics module. +function manufactured_sources(manufactured_solns_input, r_coord, z_coord, vperp_coord, + vpa_coord, vzeta_coord, vr_coord, vz_coord, composition, geometry, collisions, + num_diss_params, species) - # neutral density symbolic function - function densn_sym(Lr, Lz, r_bc, z_bc, geometry, composition, - manufactured_solns_input, species) - if manufactured_solns_input.type == "default" - if z_bc == "periodic" - if r_bc == "periodic" - densn = 1.5 + 0.1*(cos(2.0*pi*r/Lr) + cos(2.0*pi*z/Lz)) #*sin(2.0*pi*t) - elseif r_bc == "Dirichlet" - densn = 1.5 + 0.3*r/Lr - end - elseif z_bc == "wall" - T_wall = composition.T_wall - Bzed = geometry.Bzed - Bmag = geometry.Bmag - epsilon = manufactured_solns_input.epsilon_offset - alpha = manufactured_solns_input.alpha_switch - Gamma_minus = fluxconst*(Bzed/Bmag)*nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)/sqrt(pi) - Gamma_plus = fluxconst*(Bzed/Bmag)*nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)/sqrt(pi) - # exact integral of corresponding dfnn below - if composition.use_test_neutral_wall_pdf - #test - prefactor = 2.0/sqrt(pi*T_wall) - else - #proper prefactor - prefactor = 3.0*sqrt(pi)/(4.0*sqrt(T_wall)) + time_independent_sources, Source_i_func, Source_n_func = + manufactured_sources_setup(manufactured_solns_input, r_coord, z_coord, + vperp_coord, vpa_coord, vzeta_coord, vr_coord, vz_coord, composition, + geometry, collisions, num_diss_params, species) + + if time_independent_sources + # Time independent, so store arrays instead of functions + + Source_i_array = allocate_shared_float(vpa_coord.n,vperp_coord.n,z_coord.n,r_coord.n) + begin_s_r_z_region() + println("here loop thing ", looping.loop_ranges[].s) + @loop_s is begin + if is == 1 + @loop_r_z_vperp_vpa ir iz ivperp ivpa begin + Source_i_array[ivpa,ivperp,iz,ir,is] = Source_i_func(vpa_coord.grid[ivpa],vperp_coord.grid[ivperp],z_coord.grid[iz],r_coord.grid[ir],0.0) end - densn = prefactor*(Gamma_minus*(0.5 - z/Lz)^2 + Gamma_plus*(0.5 + z/Lz)^2 + 2.0 ) - else - densn = 1.0 end - elseif manufactured_solns_input.type == "2D-instability" - densn = 1.5 + 0.1*(cos(2.0*pi*r/Lr) + cos(2.0*pi*z/Lz)) - else - error("Unrecognized option " - * "manufactured_solns:type=$(manufactured_solns_input.type)") end - return densn - end - # neutral distribution symbolic function - function dfnn_sym(Lr, Lz, r_bc, z_bc, geometry, composition, manufactured_solns_input, - species) - densn = densn_sym(Lr, Lz, r_bc, z_bc, geometry, composition, - manufactured_solns_input, species) - if z_bc == "periodic" - dfnn = densn * exp( - vz^2 - vr^2 - vzeta^2) - elseif z_bc == "wall" - Hplus = 0.5*(sign(vz) + 1.0) - Hminus = 0.5*(sign(-vz) + 1.0) - FKw = knudsen_cosine(composition) - Bzed = geometry.Bzed - Bmag = geometry.Bmag - epsilon = manufactured_solns_input.epsilon_offset - alpha = manufactured_solns_input.alpha_switch - Gamma_minus = fluxconst*(Bzed/Bmag)*nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)/sqrt(pi) - Gamma_plus = fluxconst*(Bzed/Bmag)*nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)/sqrt(pi) - dfnn = Hplus *( Gamma_minus*( 0.5 - z/Lz)^2 + 1.0 )*FKw + Hminus*( Gamma_plus*( 0.5 + z/Lz)^2 + 1.0 )*FKw - end - return dfnn - end - function gyroaveraged_dfnn_sym(Lr, Lz, r_bc, z_bc, geometry, composition, - manufactured_solns_input, species) - densn = densn_sym(Lr, Lz, r_bc, z_bc, geometry, composition, - manufactured_solns_input, species) - #if (r_bc == "periodic" && z_bc == "periodic") - dfnn = densn * exp( - vpa^2 - vperp^2 ) - #end - return dfnn - end - - # ion density symbolic function - function densi_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, species) - if manufactured_solns_input.type == "default" - if z_bc == "periodic" - if r_bc == "periodic" - densi = 1.5 + 0.1*(sin(2.0*pi*r/Lr) + sin(2.0*pi*z/Lz))#*sin(2.0*pi*t) - elseif r_bc == "Dirichlet" - #densi = 1.0 + 0.5*sin(2.0*pi*z/Lz)*(r/Lr + 0.5) + 0.2*sin(2.0*pi*r/Lr)*sin(2.0*pi*t) - #densi = 1.0 + 0.5*sin(2.0*pi*z/Lz)*(r/Lr + 0.5) + sin(2.0*pi*r/Lr)*sin(2.0*pi*t) - densi = 1.0 + 0.5*(r/Lr)*sin(2.0*pi*z/Lz) + if composition.n_neutral_species > 0 + Source_n_array = allocate_shared_float(vz_coord.n,vr_coord.n,vzeta_coord.n,z_coord.n,r_coord.n) + begin_sn_r_z_region() + @loop_sn isn begin + if isn == 1 + @loop_r_z_vzeta_vr_vz ir iz ivzeta ivr ivz begin + Source_n_array[ivz,ivr,ivzeta,iz,ir,isn] = Source_n_func(vz_coord.grid[ivz],vr_coord.grid[ivr],vzeta_coord.grid[ivzeta],z_coord.grid[iz],r_coord.grid[ir],0.0) + end end - elseif z_bc == "wall" - epsilon = manufactured_solns_input.epsilon_offset - alpha = manufactured_solns_input.alpha_switch - densi = nconst*(0.5 - z/Lz)*nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha) + nconst*(z/Lz + 0.5)*nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha) + (z/Lz + 0.5)*(0.5 - z/Lz)*nzero_sym(Lr,Lz,r_bc,z_bc,alpha) #+ 0.5*(r/Lr + 0.5) + 0.5*(z/Lz + 0.5) end - elseif manufactured_solns_input.type == "2D-instability" - # Input for instability test - background_wavenumber = 1 + round(mk_int, - species.z_IC.temperature_phase) - initial_density = species.initial_density - density_amplitude = species.z_IC.density_amplitude - density_phase = species.z_IC.density_phase - T0 = Ti_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, - species) - eta0 = (initial_density - * (1.0 + density_amplitude - * sin(2.0*π*background_wavenumber*z/Lz - + density_phase))) - densi = eta0^((T0/(1+T0))) else - error("Unrecognized option " - * "manufactured_solns:type=$(manufactured_solns_input.type)") + Source_n_array = zeros(mk_float,0) end - return densi - end - function Ti_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, species) - if manufactured_solns_input.type == "default" - error("Ti_sym() is not used for the default case") - elseif manufactured_solns_input.type == "2D-instability" - background_wavenumber = 1 + round(mk_int, - species.z_IC.temperature_phase) - initial_temperature = species.initial_temperature - temperature_amplitude = species.z_IC.temperature_amplitude - T0 = (initial_temperature - * (1.0 + temperature_amplitude - * sin(2.0*π*background_wavenumber*z/Lz) - )) - else - error("Unrecognized option " - * "manufactured_solns:type=$(manufactured_solns_input.type)") - end - end - - # ion mean parallel flow symbolic function - function upari_sym(Lr,Lz,r_bc,z_bc,composition,geometry,nr,manufactured_solns_input,species) - if z_bc == "periodic" - upari = 0.0 - elseif z_bc == "wall" - densi = densi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species) - Er, Ez, phi = electric_fields(Lr,Lz,r_bc,z_bc,composition,nr,manufactured_solns_input,species) - rhostar = geometry.rhostar - bzed = geometry.bzed - epsilon = manufactured_solns_input.epsilon_offset - alpha = manufactured_solns_input.alpha_switch - upari = ( (fluxconst/(sqrt(pi)*densi))*((z/Lz + 0.5)*nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha) - - (0.5 - z/Lz)*nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)) - + alpha*(rhostar/(2.0*bzed))*Er ) - end - return upari - end - - # ion parallel pressure symbolic function - function ppari_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species) - # normalisation factor due to pressure normalisation convention in master pref = nref mref cref^2 - norm_fac = 0.5 - if z_bc == "periodic" - densi = densi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species) - ppari = densi - elseif z_bc == "wall" - densi = densi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species) - epsilon = manufactured_solns_input.epsilon_offset - alpha = manufactured_solns_input.alpha_switch - ppari = ( pconst*((0.5 - z/Lz)*nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha) - + (z/Lz + 0.5)*nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)) - + (z/Lz + 0.5)*(0.5 - z/Lz)*nzero_sym(Lr,Lz,r_bc,z_bc,alpha) - - (2.0/(pi*densi))*((z/Lz + 0.5)*nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha) - - (0.5 - z/Lz)*nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha))^2 ) - end - return ppari*norm_fac - end - - # ion perpendicular pressure symbolic function - function pperpi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species,nvperp) - densi = densi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species) - normfac = 0.5 # if pressure normalised to 0.5* nref * Tref = mref cref^2 - #normfac = 1.0 # if pressure normalised to nref*Tref - if nvperp > 1 - pperpi = densi # simple vperp^2 dependence of dfni - else - pperpi = 0.0 # marginalised model has nvperp = 1, vperp[1] = 0 - end - return pperpi*normfac - end - - # ion thermal speed symbolic function - function vthi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species,nvperp) - densi = densi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species) - ppari = ppari_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species) - pperpi = pperpi_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input,species,nvperp) - isotropic_pressure = (1.0/3.0)*(ppari + 2.0*pperpi) - normfac = 2.0 # if pressure normalised to 0.5* nref * Tref = mref cref^2 - #normfac = 1.0 # if pressure normalised to nref*Tref - if nvperp > 1 - vthi = sqrt(normfac*isotropic_pressure/densi) # thermal speed definition of 2V model - else - vthi = sqrt(normfac*ppari/densi) # thermal speed definition of 1V model - end - return vthi + manufactured_sources_list = (time_independent_sources = true, Source_i_array = Source_i_array, Source_n_array = Source_n_array) + else + manufactured_sources_list = (time_independent_sources = false, Source_i_func = Source_i_func, Source_n_func = Source_n_func) end - - function jpari_into_LHS_wall_sym(Lr,Lz,r_bc,z_bc,composition,manufactured_solns_input) - if z_bc == "periodic" - jpari_into_LHS_wall_sym = 0.0 - elseif z_bc == "wall" - #appropriate for wall bc test when Er = 0 (nr == 1) - epsilon = manufactured_solns_input.epsilon_offset - alpha = manufactured_solns_input.alpha_switch - jpari_into_LHS_wall_sym = -fluxconst*nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)/sqrt(pi) - end - return jpari_into_LHS_wall_sym - end - - # ion distribution symbolic function - function dfni_sym(Lr, Lz, r_bc, z_bc, composition, geometry, nr, - manufactured_solns_input, species) - densi = densi_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, - species) - - if manufactured_solns_input.type == "default" - # calculate the electric fields and the potential - Er, Ez, phi = electric_fields(Lr, Lz, r_bc, z_bc, composition, nr, - manufactured_solns_input, species) - - # get geometric/composition data - Bzed = geometry.Bzed - Bmag = geometry.Bmag - rhostar = geometry.rhostar - epsilon = manufactured_solns_input.epsilon_offset - alpha = manufactured_solns_input.alpha_switch - if z_bc == "periodic" - dfni = densi * exp( - vpa^2 - vperp^2) - elseif z_bc == "wall" - vpabar = vpa - alpha*(rhostar/2.0)*(Bmag/Bzed)*Er # for alpha = 1.0, effective velocity in z direction * (Bmag/Bzed) - Hplus = 0.5*(sign(vpabar) + 1.0) - Hminus = 0.5*(sign(-vpabar) + 1.0) - ffa = exp(- vperp^2) - dfni = ffa * ( nminus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)* (0.5 - z/Lz) * Hminus * vpabar^pvpa + nplus_sym(Lr,Lz,r_bc,z_bc,epsilon,alpha)*(z/Lz + 0.5) * Hplus * vpabar^pvpa + nzero_sym(Lr,Lz,r_bc,z_bc,alpha)*(z/Lz + 0.5)*(0.5 - z/Lz) ) * exp( - vpabar^2 ) - end - elseif manufactured_solns_input.type == "2D-instability" - # Input for instability test - T0 = Ti_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, - species) - vth = sqrt(T0) - # Note this is for a '1V' test - dfni = densi/vth * exp(-(vpa/vth)^2) - else - error("Unrecognized option " - * "manufactured_solns:type=$(manufactured_solns_input.type)") - end - return dfni - end - function cartesian_dfni_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, - species) - densi = densi_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, - species) - #if (r_bc == "periodic" && z_bc == "periodic") || (r_bc == "Dirichlet" && z_bc == "periodic") - dfni = densi * exp( - vz^2 - vr^2 - vzeta^2) - #end - return dfni - end - - function electric_fields(Lr, Lz, r_bc, z_bc, composition, nr, - manufactured_solns_input, species) - - # define derivative operators - Dr = Differential(r) - Dz = Differential(z) - - # get N_e factor for boltzmann response - if composition.electron_physics == boltzmann_electron_response_with_simple_sheath && nr == 1 - # so 1D MMS test with 3V neutrals where ion current can be calculated prior to knowing Er - jpari_into_LHS_wall = jpari_into_LHS_wall_sym(Lr, Lz, r_bc, z_bc, - manufactured_solns_input) - N_e = -2.0*sqrt(pi*composition.me_over_mi)*exp(-composition.phi_wall/composition.T_e)*jpari_into_LHS_wall - elseif composition.electron_physics == boltzmann_electron_response_with_simple_sheath && nr > 1 - println("ERROR: simple sheath MMS test not supported for nr > 1") - println("INFO: In general, not possible to analytically calculate jpari for sheath prior to knowing Er, but Er depends on jpari") - println("Setting N_e = 1.0. Expect MMS test to fail!") - N_e = 1.0 - elseif composition.electron_physics == boltzmann_electron_response - # all other cases - # N_e equal to reference density - N_e = 1.0 - end - - if nr > 1 # keep radial electric field - rfac = 1.0 - else # drop radial electric field - rfac = 0.0 - end - - densi = densi_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, - species) - # calculate the electric fields - dense = densi # get the electron density via quasineutrality with Zi = 1 - phi = composition.T_e*log(dense/N_e) # use the adiabatic response of electrons for me/mi -> 0 - Er = -Dr(phi)*rfac + composition.Er_constant - Ez = -Dz(phi) - - Er_expanded = expand_derivatives(Er) - Ez_expanded = expand_derivatives(Ez) - - return Er_expanded, Ez_expanded, phi - end - - function manufactured_solutions(manufactured_solns_input, Lr, Lz, r_bc, z_bc, - geometry, composition, species, nr, nvperp) - charged_species = species.charged[1] - if composition.n_neutral_species > 0 - neutral_species = species.neutral[1] - else - neutral_species = nothing - end - - densi = densi_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, - charged_species) - upari = upari_sym(Lr, Lz, r_bc, z_bc, composition, geometry, nr, manufactured_solns_input, - charged_species) - ppari = ppari_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, - charged_species) - pperpi = pperpi_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, - charged_species, nvperp) - vthi = vthi_sym(Lr, Lz, r_bc, z_bc, composition, manufactured_solns_input, - charged_species, nvperp) - dfni = dfni_sym(Lr, Lz, r_bc, z_bc, composition, geometry, nr, - manufactured_solns_input, charged_species) - - densn = densn_sym(Lr, Lz, r_bc, z_bc, geometry,composition, - manufactured_solns_input, neutral_species) - dfnn = dfnn_sym(Lr, Lz, r_bc, z_bc, geometry, composition, - manufactured_solns_input, neutral_species) - - #build julia functions from these symbolic expressions - # cf. https://docs.juliahub.com/Symbolics/eABRO/3.4.0/tutorials/symbolic_functions/ - densi_func = build_function(densi, z, r, t, expression=Val{false}) - upari_func = build_function(upari, z, r, t, expression=Val{false}) - ppari_func = build_function(ppari, z, r, t, expression=Val{false}) - pperpi_func = build_function(pperpi, z, r, t, expression=Val{false}) - vthi_func = build_function(vthi, z, r, t, expression=Val{false}) - densn_func = build_function(densn, z, r, t, expression=Val{false}) - dfni_func = build_function(dfni, vpa, vperp, z, r, t, expression=Val{false}) - dfnn_func = build_function(dfnn, vz, vr, vzeta, z, r, t, expression=Val{false}) - # return function - # call like: - # densi_func(zval, rval, tval) - # dfni_func(vpaval, vperpval, zval, rval, tval) - # densn_func(zval, rval, tval) - # dfnn_func(vzval, vrval, vzetapval, zval, rval, tval) - - manufactured_solns_list = (densi_func = densi_func, densn_func = densn_func, - dfni_func = dfni_func, dfnn_func = dfnn_func, - upari_func = upari_func, ppari_func = ppari_func, - pperpi_func = pperpi_func, vthi_func = vthi_func) - - return manufactured_solns_list - end - - function manufactured_electric_fields(Lr, Lz, r_bc, z_bc, composition, nr, - manufactured_solns_input, species) - - # calculate the electric fields and the potential - Er, Ez, phi = electric_fields(Lr, Lz, r_bc, z_bc, composition, nr, - manufactured_solns_input, species) - - Er_func = build_function(Er, z, r, t, expression=Val{false}) - Ez_func = build_function(Ez, z, r, t, expression=Val{false}) - phi_func = build_function(phi, z, r, t, expression=Val{false}) - - manufactured_E_fields = (Er_func = Er_func, Ez_func = Ez_func, phi_func = phi_func) - - return manufactured_E_fields - end - - function manufactured_sources(manufactured_solns_input, r_coord, z_coord, vperp_coord, - vpa_coord, vzeta_coord, vr_coord, vz_coord, composition, geometry, collisions, - num_diss_params, species) - - charged_species = species.charged[1] - if composition.n_neutral_species > 0 - neutral_species = species.neutral[1] - else - neutral_species = nothing - end - - # ion manufactured solutions - densi = densi_sym(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, composition, - manufactured_solns_input, charged_species) - upari = upari_sym(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, composition, geometry, r_coord.n, manufactured_solns_input, charged_species) - vthi = vthi_sym(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, composition, manufactured_solns_input, - charged_species, vperp_coord.n) - dfni = dfni_sym(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, composition, - geometry, r_coord.n, manufactured_solns_input, charged_species) - #dfni in vr vz vzeta coordinates - vrvzvzeta_dfni = cartesian_dfni_sym(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, - composition, manufactured_solns_input, - charged_species) - - # neutral manufactured solutions - densn = densn_sym(r_coord.L,z_coord.L, r_coord.bc, z_coord.bc, geometry, - composition, manufactured_solns_input, neutral_species) - dfnn = dfnn_sym(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, geometry, - composition, manufactured_solns_input, neutral_species) - # gyroaverage < dfnn > in vpa vperp coordinates - gav_dfnn = gyroaveraged_dfnn_sym(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, - geometry, composition, manufactured_solns_input, - neutral_species) - - dense = densi # get the electron density via quasineutrality with Zi = 1 - - # define derivative operators - Dr = Differential(r) - Dz = Differential(z) - Dvpa = Differential(vpa) - Dvperp = Differential(vperp) - Dvz = Differential(vz) - Dt = Differential(t) - - # get geometric/composition data - Bzed = geometry.Bzed - Bmag = geometry.Bmag - rhostar = geometry.rhostar - #exceptions for cases with missing terms - if composition.n_neutral_species > 0 - cx_frequency = collisions.charge_exchange - ionization_frequency = collisions.ionization - else - cx_frequency = 0.0 - ionization_frequency = 0.0 - end - if r_coord.n > 1 # keep radial derivatives - rfac = 1.0 - else # drop radial derivative terms - rfac = 0.0 - end - - # calculate the electric fields and the potential - Er, Ez, phi = electric_fields(r_coord.L, z_coord.L, r_coord.bc, z_coord.bc, - composition, r_coord.n, manufactured_solns_input, - charged_species) - - # the ion source to maintain the manufactured solution - Si = ( Dt(dfni) + ( vpa * (Bzed/Bmag) - 0.5*rhostar*Er ) * Dz(dfni) + ( 0.5*rhostar*Ez*rfac ) * Dr(dfni) + ( 0.5*Ez*Bzed/Bmag ) * Dvpa(dfni) - + cx_frequency*( densn*dfni - densi*gav_dfnn ) - ionization_frequency*dense*gav_dfnn) - nu_krook = collisions.krook_collision_frequency_prefactor - if nu_krook > 0.0 - Ti_over_Tref = vthi^2 - if collisions.krook_collisions_option == "manual" - nuii_krook = nu_krook - else # default option - nuii_krook = nu_krook * densi * Ti_over_Tref^(-1.5) - end - if vperp_coord.n > 1 - pvth = 3 - else - pvth = 1 - end - FMaxwellian = (densi/vthi^pvth)*exp( -( ( vpa-upari)^2 + vperp^2 )/vthi^2) - Si += - nuii_krook*(FMaxwellian - dfni) - end - include_num_diss_in_MMS = true - if num_diss_params.vpa_dissipation_coefficient > 0.0 && include_num_diss_in_MMS - Si += - num_diss_params.vpa_dissipation_coefficient*Dvpa(Dvpa(dfni)) - end - if num_diss_params.vperp_dissipation_coefficient > 0.0 && include_num_diss_in_MMS - Si += - num_diss_params.vperp_dissipation_coefficient*Dvperp(Dvperp(dfni)) - end - if num_diss_params.r_dissipation_coefficient > 0.0 && include_num_diss_in_MMS - Si += - rfac*num_diss_params.r_dissipation_coefficient*Dr(Dr(dfni)) - end - if num_diss_params.z_dissipation_coefficient > 0.0 && include_num_diss_in_MMS - Si += - num_diss_params.z_dissipation_coefficient*Dz(Dz(dfni)) - end - - Source_i = expand_derivatives(Si) - - # the neutral source to maintain the manufactured solution - Sn = Dt(dfnn) + vz * Dz(dfnn) + rfac*vr * Dr(dfnn) + cx_frequency* (densi*dfnn - densn*vrvzvzeta_dfni) + ionization_frequency*dense*dfnn - if num_diss_params.vz_dissipation_coefficient > 0.0 && include_num_diss_in_MMS - Sn += - num_diss_params.vz_dissipation_coefficient*Dvz(Dvz(dfnn)) - end - if num_diss_params.r_dissipation_coefficient > 0.0 && include_num_diss_in_MMS - Sn += - rfac*num_diss_params.r_dissipation_coefficient*Dr(Dr(dfnn)) - end - if num_diss_params.z_dissipation_coefficient > 0.0 && include_num_diss_in_MMS - Sn += - num_diss_params.z_dissipation_coefficient*Dz(Dz(dfnn)) - end - - Source_n = expand_derivatives(Sn) - - Source_i_func = build_function(Source_i, vpa, vperp, z, r, t, expression=Val{false}) - Source_n_func = build_function(Source_n, vz, vr, vzeta, z, r, t, expression=Val{false}) - - if expand_derivatives(Dt(Source_i)) == 0 && expand_derivatives(Dt(Source_n)) == 0 - # Time independent, so store arrays instead of functions - - Source_i_array = allocate_shared_float(vpa_coord.n,vperp_coord.n,z_coord.n,r_coord.n) - begin_s_r_z_region() - @loop_s is begin - if is == 1 - @loop_r_z_vperp_vpa ir iz ivperp ivpa begin - Source_i_array[ivpa,ivperp,iz,ir,is] = Source_i_func(vpa_coord.grid[ivpa],vperp_coord.grid[ivperp],z_coord.grid[iz],r_coord.grid[ir],0.0) - end - end - end - - if composition.n_neutral_species > 0 - Source_n_array = allocate_shared_float(vz_coord.n,vr_coord.n,vzeta_coord.n,z_coord.n,r_coord.n) - begin_sn_r_z_region() - @loop_sn isn begin - if isn == 1 - @loop_r_z_vzeta_vr_vz ir iz ivzeta ivr ivz begin - Source_n_array[ivz,ivr,ivzeta,iz,ir,isn] = Source_n_func(vz_coord.grid[ivz],vr_coord.grid[ivr],vzeta_coord.grid[ivzeta],z_coord.grid[iz],r_coord.grid[ir],0.0) - end - end - end - else - Source_n_array = zeros(mk_float,0) - end + return manufactured_sources_list +end - manufactured_sources_list = (time_independent_sources = true, Source_i_array = Source_i_array, Source_n_array = Source_n_array) - else - manufactured_sources_list = (time_independent_sources = false, Source_i_func = Source_i_func, Source_n_func = Source_n_func) - end - - return manufactured_sources_list - end - end diff --git a/moment_kinetics/src/moment_kinetics_input.jl b/moment_kinetics/src/moment_kinetics_input.jl index 586bbbe9b..47f71edfb 100644 --- a/moment_kinetics/src/moment_kinetics_input.jl +++ b/moment_kinetics/src/moment_kinetics_input.jl @@ -219,6 +219,16 @@ function mk_input(scan_input=Dict(); save_inputs_to_txt=false, ignore_MPI=true) # manufactured_solns_section["use_for_init"] == true manufactured_solns_section["use_for_init"] = true end + if manufactured_solns_section["use_for_init"] || manufactured_solns_section["use_for_advance"] + manufactured_solutions_ext = Base.get_extension(@__MODULE__, :manufactured_solns_ext) + if manufactured_solutions_ext === nothing + # If Symbolics is not installed, then the extension manufactured_solns_ext + # will not be loaded, in which case we cannot use manufactured solutions. + error("Symbolics package is not installed, so manufactured solutions are not " + * "available. Re-run machines/machine-setup.sh and activate " + * "manufactured solutions, or install Symbolics.") + end + end if manufactured_solns_section["use_vpabar_in_mms_dfni"] manufactured_solns_section["alpha_switch"] = 1.0 else From d1deab48d1bbddc5e16136b921b390cce26927e2 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 19 Dec 2023 21:44:21 +0000 Subject: [PATCH 10/80] Move functions to analysis.jl so fit_delta_phi_mode can be used in tests --- makie_post_processing/src/shared_utils.jl | 1 + moment_kinetics/Project.toml | 1 + moment_kinetics/src/analysis.jl | 219 ++++++++++++++++++ moment_kinetics/test/sound_wave_tests.jl | 2 +- .../src/plots_post_processing.jl | 218 ----------------- 5 files changed, 222 insertions(+), 219 deletions(-) diff --git a/makie_post_processing/src/shared_utils.jl b/makie_post_processing/src/shared_utils.jl index d4b33e465..700a24b24 100644 --- a/makie_post_processing/src/shared_utils.jl +++ b/makie_post_processing/src/shared_utils.jl @@ -3,6 +3,7 @@ module shared_utils export calculate_and_write_frequencies, construct_global_zr_coords, get_geometry_and_composition, read_distributed_zr_data! +using moment_kinetics.analysis: fit_delta_phi_mode using moment_kinetics.coordinates: define_coordinate using moment_kinetics.input_structs: grid_input using moment_kinetics.type_definitions: mk_float, mk_int diff --git a/moment_kinetics/Project.toml b/moment_kinetics/Project.toml index 2b4d2f3fb..ed20aba82 100644 --- a/moment_kinetics/Project.toml +++ b/moment_kinetics/Project.toml @@ -16,6 +16,7 @@ Interpolations = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59" LegendrePolynomials = "3db4a2ba-fc88-11e8-3e01-49c72059a882" LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +LsqFit = "2fda8390-95c7-5789-9bda-21331edee243" MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" Measures = "442fdcdd-2543-5da2-b0f3-8c86c306513e" diff --git a/moment_kinetics/src/analysis.jl b/moment_kinetics/src/analysis.jl index bfc21d5a6..431a346a1 100644 --- a/moment_kinetics/src/analysis.jl +++ b/moment_kinetics/src/analysis.jl @@ -19,6 +19,7 @@ using ..type_definitions: mk_int using ..velocity_moments: integrate_over_vspace using FFTW +using LsqFit using MPI using OrderedCollections using Statistics @@ -830,4 +831,222 @@ function get_unnormalised_f_2d(f, density, vth, evolve_density, evolve_ppar) return f_unnorm end +""" +Calculate a moving average + +``` +result[i] = mean(v[i-n:i+n]) +``` +Except near the ends of the array where indices outside the range of v are skipped. +""" +function moving_average(v::AbstractVector, n::mk_int) + if length(v) < 2*n+1 + error("Cannot take moving average with n=$n on vector of length=$(length(v))") + end + result = similar(v) + for i ∈ 1:n + result[i] = mean(v[begin:i+n]) + end + for i ∈ n+1:length(v)-n-1 + result[i] = mean(v[i-n:i+n]) + end + for i ∈ length(v)-n:length(v) + result[i] = mean(v[i-n:end]) + end + return result +end + +""" +Fit delta_phi to get the frequency and growth rate. + +Note, expect the input to be a standing wave (as simulations are initialised with just a +density perturbation), so need to extract both frequency and growth rate from the +time-variation of the amplitude. + +The function assumes that if the amplitude does not cross zero, then the mode is +non-oscillatory and so fits just an exponential, not exp*cos. The simulation used as +input should be long enough to contain at least ~1 period of oscillation if the mode is +oscillatory or the fit will not work. + +Arguments +--------- +z : Array{mk_float, 1} + 1d array of the grid point positions +t : Array{mk_float, 1} + 1d array of the time points +delta_phi : Array{mk_float, 2} + 2d array of the values of delta_phi(z, t) + +Returns +------- +phi_fit_result struct whose fields are: + growth_rate : mk_float + Fitted growth rate of the mode + amplitude0 : mk_float + Fitted amplitude at t=0 + frequency : mk_float + Fitted frequency of the mode + offset0 : mk_float + Fitted offset at t=0 + amplitude_fit_error : mk_float + RMS error in fit to ln(amplitude) - i.e. ln(A) + offset_fit_error : mk_float + RMS error in fit to offset - i.e. δ + cosine_fit_error : mk_float + Maximum of the RMS errors of the cosine fits at each time point + amplitude : Array{mk_float, 1} + Values of amplitude from which growth_rate fit was calculated + offset : Array{mk_float, 1} + Values of offset from which frequency fit was calculated +""" +function fit_delta_phi_mode(t, z, delta_phi) + # First fit a cosine to each time slice + results = allocate_float(3, size(delta_phi)[2]) + amplitude_guess = 1.0 + offset_guess = 0.0 + for (i, phi_z) in enumerate(eachcol(delta_phi)) + results[:, i] .= fit_cosine(z, phi_z, amplitude_guess, offset_guess) + (amplitude_guess, offset_guess) = results[1:2, i] + end + + amplitude = results[1, :] + offset = results[2, :] + cosine_fit_error = results[3, :] + + L = z[end] - z[begin] + + # Choose initial amplitude to be positive, for convenience. + if amplitude[1] < 0 + # 'Wrong sign' of amplitude is equivalent to a phase shift by π + amplitude .*= -1.0 + offset .+= L / 2.0 + end + + # model for linear fits + @. model(t, p) = p[1] * t + p[2] + + # Fit offset vs. time + # Would give phase velocity for a travelling wave, but we expect either a standing + # wave or a zero-frequency decaying mode, so expect the time variation of the offset + # to be ≈0 + offset_fit = curve_fit(model, t, offset, [1.0, 0.0]) + doffsetdt = offset_fit.param[1] + offset0 = offset_fit.param[2] + offset_error = sqrt(mean(offset_fit.resid .^ 2)) + offset_tol = 2.e-5 + if abs(doffsetdt) > offset_tol + println("WARNING: d(offset)/dt=", doffsetdt, " is non-negligible (>", offset_tol, + ") but fit_delta_phi_mode expected either a standing wave or a ", + "zero-frequency decaying mode.") + end + + growth_rate = 0.0 + amplitude0 = 0.0 + frequency = 0.0 + phase = 0.0 + fit_error = 0.0 + if all(amplitude .> 0.0) + # No zero crossing, so assume the mode is non-oscillatory (i.e. purely + # growing/decaying). + + # Fit ln(amplitude) vs. time so we don't give extra weight to early time points + amplitude_fit = curve_fit(model, t, log.(amplitude), [-1.0, 1.0]) + growth_rate = amplitude_fit.param[1] + amplitude0 = exp(amplitude_fit.param[2]) + fit_error = sqrt(mean(amplitude_fit.resid .^ 2)) + frequency = 0.0 + phase = 0.0 + else + converged = false + maxiter = 100 + for iter ∈ 1:maxiter + @views growth_rate_change, frequency, phase, fit_error = + fit_phi0_vs_time(exp.(-growth_rate*t) .* amplitude, t) + growth_rate += growth_rate_change + println("growth_rate: ", growth_rate, " growth_rate_change/growth_rate: ", growth_rate_change/growth_rate, " fit_error: ", fit_error) + if abs(growth_rate_change/growth_rate) < 1.0e-12 || fit_error < 1.0e-11 + converged = true + break + end + end + if !converged + println("WARNING: Iteration to find growth rate failed to converge in ", maxiter, " iterations") + end + amplitude0 = amplitude[1] / cos(phase) + end + + return (growth_rate=growth_rate, frequency=frequency, phase=phase, + amplitude0=amplitude0, offset0=offset0, amplitude_fit_error=fit_error, + offset_fit_error=offset_error, cosine_fit_error=maximum(cosine_fit_error), + amplitude=amplitude, offset=offset) +end + +""" +Fit a cosine to a 1d array + +Fit function is A*cos(2*π*n*(z + δ)/L) + +The domain z is taken to be periodic, with the first and last points identified, so +L=z[end]-z[begin] + +Arguments +--------- +z : Array + 1d array with positions of the grid points - should have the same length as data +data : Array + 1d array of the data to be fit +amplitude_guess : Float + Initial guess for the amplitude (the value from the previous time point might be a + good choice) +offset_guess : Float + Initial guess for the offset (the value from the previous time point might be a good + choice) +n : Int, default 1 + The periodicity used for the fit + +Returns +------- +amplitude : Float + The amplitude A of the cosine fit +offset : Float + The offset δ of the cosine fit +error : Float + The RMS of the difference between data and the fit +""" +function fit_cosine(z, data, amplitude_guess, offset_guess, n=1) + # Length of domain + L = z[end] - z[begin] + + @. model(z, p) = p[1] * cos(2*π*n*(z + p[2])/L) + fit = curve_fit(model, z, data, [amplitude_guess, offset_guess]) + + # calculate error + error = sqrt(mean(fit.resid .^ 2)) + + return fit.param[1], fit.param[2], error +end + +function fit_phi0_vs_time(phi0, tmod) + # the model we are fitting to the data is given by the function 'model': + # assume phi(z0,t) = exp(γt)cos(ωt+φ) so that + # phi(z0,t)/phi(z0,t0) = exp((t-t₀)γ)*cos((t-t₀)*ω + phase)/cos(phase), + # where tmod = t-t0 and phase = ωt₀-φ + @. model(t, p) = exp(p[1]*t) * cos(p[2]*t + p[3]) / cos(p[3]) + model_params = allocate_float(3) + model_params[1] = -0.1 + model_params[2] = 8.6 + model_params[3] = 0.0 + @views fit = curve_fit(model, tmod, phi0/phi0[1], model_params) + # get the confidence interval at 10% level for each fit parameter + #se = standard_error(fit) + #standard_deviation = Array{Float64,1} + #@. standard_deviation = se * sqrt(size(tmod)) + + fitted_function = model(tmod, fit.param) + norm = moving_average(@.((abs(phi0/phi0[1]) + abs(fitted_function))^2), 1) + fit_error = sqrt(mean(@.((phi0/phi0[1] - fitted_function)^2 / norm))) + + return fit.param[1], fit.param[2], fit.param[3], fit_error +end + end diff --git a/moment_kinetics/test/sound_wave_tests.jl b/moment_kinetics/test/sound_wave_tests.jl index 21c187288..4d6ea933e 100644 --- a/moment_kinetics/test/sound_wave_tests.jl +++ b/moment_kinetics/test/sound_wave_tests.jl @@ -10,7 +10,7 @@ using moment_kinetics.load_data: open_readonly_output_file using moment_kinetics.load_data: load_fields_data, load_time_data using moment_kinetics.load_data: load_species_data, load_coordinate_data using moment_kinetics.analysis: analyze_fields_data -using moment_kinetics.post_processing: fit_delta_phi_mode +using moment_kinetics.analysis: fit_delta_phi_mode const analytical_rtol = 3.e-2 const regression_rtol = 1.e-14 diff --git a/plots_post_processing/src/plots_post_processing.jl b/plots_post_processing/src/plots_post_processing.jl index e9ccbe8c7..9bf57ece8 100644 --- a/plots_post_processing/src/plots_post_processing.jl +++ b/plots_post_processing/src/plots_post_processing.jl @@ -85,31 +85,6 @@ function __init__() global_size[] == 0 && pyplot() end -""" -Calculate a moving average - -``` -result[i] = mean(v[i-n:i+n]) -``` -Except near the ends of the array where indices outside the range of v are skipped. -""" -function moving_average(v::AbstractVector, n::mk_int) - if length(v) < 2*n+1 - error("Cannot take moving average with n=$n on vector of length=$(length(v))") - end - result = similar(v) - for i ∈ 1:n - result[i] = mean(v[begin:i+n]) - end - for i ∈ n+1:length(v)-n-1 - result[i] = mean(v[i-n:i+n]) - end - for i ∈ length(v)-n:length(v) - result[i] = mean(v[i-n:end]) - end - return result -end - """ Call savefig, but catch the exception if there is an error """ @@ -2486,199 +2461,6 @@ function plot_dfns(density, parallel_flow, parallel_pressure, thermal_speed, ff, return nothing end -""" -Fit delta_phi to get the frequency and growth rate. - -Note, expect the input to be a standing wave (as simulations are initialised with just a -density perturbation), so need to extract both frequency and growth rate from the -time-variation of the amplitude. - -The function assumes that if the amplitude does not cross zero, then the mode is -non-oscillatory and so fits just an exponential, not exp*cos. The simulation used as -input should be long enough to contain at least ~1 period of oscillation if the mode is -oscillatory or the fit will not work. - -Arguments ---------- -z : Array{mk_float, 1} - 1d array of the grid point positions -t : Array{mk_float, 1} - 1d array of the time points -delta_phi : Array{mk_float, 2} - 2d array of the values of delta_phi(z, t) - -Returns -------- -phi_fit_result struct whose fields are: - growth_rate : mk_float - Fitted growth rate of the mode - amplitude0 : mk_float - Fitted amplitude at t=0 - frequency : mk_float - Fitted frequency of the mode - offset0 : mk_float - Fitted offset at t=0 - amplitude_fit_error : mk_float - RMS error in fit to ln(amplitude) - i.e. ln(A) - offset_fit_error : mk_float - RMS error in fit to offset - i.e. δ - cosine_fit_error : mk_float - Maximum of the RMS errors of the cosine fits at each time point - amplitude : Array{mk_float, 1} - Values of amplitude from which growth_rate fit was calculated - offset : Array{mk_float, 1} - Values of offset from which frequency fit was calculated -""" -function fit_delta_phi_mode(t, z, delta_phi) - # First fit a cosine to each time slice - results = allocate_float(3, size(delta_phi)[2]) - amplitude_guess = 1.0 - offset_guess = 0.0 - for (i, phi_z) in enumerate(eachcol(delta_phi)) - results[:, i] .= fit_cosine(z, phi_z, amplitude_guess, offset_guess) - (amplitude_guess, offset_guess) = results[1:2, i] - end - - amplitude = results[1, :] - offset = results[2, :] - cosine_fit_error = results[3, :] - - L = z[end] - z[begin] - - # Choose initial amplitude to be positive, for convenience. - if amplitude[1] < 0 - # 'Wrong sign' of amplitude is equivalent to a phase shift by π - amplitude .*= -1.0 - offset .+= L / 2.0 - end - - # model for linear fits - @. model(t, p) = p[1] * t + p[2] - - # Fit offset vs. time - # Would give phase velocity for a travelling wave, but we expect either a standing - # wave or a zero-frequency decaying mode, so expect the time variation of the offset - # to be ≈0 - offset_fit = curve_fit(model, t, offset, [1.0, 0.0]) - doffsetdt = offset_fit.param[1] - offset0 = offset_fit.param[2] - offset_error = sqrt(mean(offset_fit.resid .^ 2)) - offset_tol = 2.e-5 - if abs(doffsetdt) > offset_tol - println("WARNING: d(offset)/dt=", doffsetdt, " is non-negligible (>", offset_tol, - ") but fit_delta_phi_mode expected either a standing wave or a ", - "zero-frequency decaying mode.") - end - - growth_rate = 0.0 - amplitude0 = 0.0 - frequency = 0.0 - phase = 0.0 - fit_error = 0.0 - if all(amplitude .> 0.0) - # No zero crossing, so assume the mode is non-oscillatory (i.e. purely - # growing/decaying). - - # Fit ln(amplitude) vs. time so we don't give extra weight to early time points - amplitude_fit = curve_fit(model, t, log.(amplitude), [-1.0, 1.0]) - growth_rate = amplitude_fit.param[1] - amplitude0 = exp(amplitude_fit.param[2]) - fit_error = sqrt(mean(amplitude_fit.resid .^ 2)) - frequency = 0.0 - phase = 0.0 - else - converged = false - maxiter = 100 - for iter ∈ 1:maxiter - @views growth_rate_change, frequency, phase, fit_error = - fit_phi0_vs_time(exp.(-growth_rate*t) .* amplitude, t) - growth_rate += growth_rate_change - println("growth_rate: ", growth_rate, " growth_rate_change/growth_rate: ", growth_rate_change/growth_rate, " fit_error: ", fit_error) - if abs(growth_rate_change/growth_rate) < 1.0e-12 || fit_error < 1.0e-11 - converged = true - break - end - end - if !converged - println("WARNING: Iteration to find growth rate failed to converge in ", maxiter, " iterations") - end - amplitude0 = amplitude[1] / cos(phase) - end - - return (growth_rate=growth_rate, frequency=frequency, phase=phase, - amplitude0=amplitude0, offset0=offset0, amplitude_fit_error=fit_error, - offset_fit_error=offset_error, cosine_fit_error=maximum(cosine_fit_error), - amplitude=amplitude, offset=offset) -end - -function fit_phi0_vs_time(phi0, tmod) - # the model we are fitting to the data is given by the function 'model': - # assume phi(z0,t) = exp(γt)cos(ωt+φ) so that - # phi(z0,t)/phi(z0,t0) = exp((t-t₀)γ)*cos((t-t₀)*ω + phase)/cos(phase), - # where tmod = t-t0 and phase = ωt₀-φ - @. model(t, p) = exp(p[1]*t) * cos(p[2]*t + p[3]) / cos(p[3]) - model_params = allocate_float(3) - model_params[1] = -0.1 - model_params[2] = 8.6 - model_params[3] = 0.0 - @views fit = curve_fit(model, tmod, phi0/phi0[1], model_params) - # get the confidence interval at 10% level for each fit parameter - #se = standard_error(fit) - #standard_deviation = Array{Float64,1} - #@. standard_deviation = se * sqrt(size(tmod)) - - fitted_function = model(tmod, fit.param) - norm = moving_average(@.((abs(phi0/phi0[1]) + abs(fitted_function))^2), 1) - fit_error = sqrt(mean(@.((phi0/phi0[1] - fitted_function)^2 / norm))) - - return fit.param[1], fit.param[2], fit.param[3], fit_error -end - -""" -Fit a cosine to a 1d array - -Fit function is A*cos(2*π*n*(z + δ)/L) - -The domain z is taken to be periodic, with the first and last points identified, so -L=z[end]-z[begin] - -Arguments ---------- -z : Array - 1d array with positions of the grid points - should have the same length as data -data : Array - 1d array of the data to be fit -amplitude_guess : Float - Initial guess for the amplitude (the value from the previous time point might be a - good choice) -offset_guess : Float - Initial guess for the offset (the value from the previous time point might be a good - choice) -n : Int, default 1 - The periodicity used for the fit - -Returns -------- -amplitude : Float - The amplitude A of the cosine fit -offset : Float - The offset δ of the cosine fit -error : Float - The RMS of the difference between data and the fit -""" -function fit_cosine(z, data, amplitude_guess, offset_guess, n=1) - # Length of domain - L = z[end] - z[begin] - - @. model(z, p) = p[1] * cos(2*π*n*(z + p[2])/L) - fit = curve_fit(model, z, data, [amplitude_guess, offset_guess]) - - # calculate error - error = sqrt(mean(fit.resid .^ 2)) - - return fit.param[1], fit.param[2], error -end - #function advection_test_1d(fstart, fend) # rmserr = sqrt(sum((fend .- fstart).^2))/(size(fend,1)*size(fend,2)*size(fend,3)) # println("advection_test_1d rms error: ", rmserr) From 6f51d7c858097816c3c211ab4487195d95574307 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 19 Dec 2023 22:07:33 +0000 Subject: [PATCH 11/80] Move data-loading functions from makie_post_processing to load_data This lets us use them in the tests. --- makie_post_processing/Project.toml | 1 - .../src/makie_post_processing.jl | 643 +--------------- makie_post_processing/src/shared_utils.jl | 96 +-- moment_kinetics/src/load_data.jl | 726 +++++++++++++++++- moment_kinetics/src/moment_kinetics.jl | 2 +- .../test/restart_interpolation_tests.jl | 6 +- .../src/plot_MMS_sequence.jl | 2 +- plots_post_processing/src/plot_sequence.jl | 2 +- .../src/plots_post_processing.jl | 7 +- 9 files changed, 759 insertions(+), 726 deletions(-) diff --git a/makie_post_processing/Project.toml b/makie_post_processing/Project.toml index 4416f6ed1..cb07db97f 100644 --- a/makie_post_processing/Project.toml +++ b/makie_post_processing/Project.toml @@ -6,7 +6,6 @@ version = "0.1.0" [deps] CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" -Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" LsqFit = "2fda8390-95c7-5789-9bda-21331edee243" MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" diff --git a/makie_post_processing/src/makie_post_processing.jl b/makie_post_processing/src/makie_post_processing.jl index 5545c4373..d78c73be3 100644 --- a/makie_post_processing/src/makie_post_processing.jl +++ b/makie_post_processing/src/makie_post_processing.jl @@ -34,20 +34,19 @@ using moment_kinetics.krook_collisions: get_collision_frequency using moment_kinetics.looping: all_dimensions, ion_dimensions, neutral_dimensions using moment_kinetics.manufactured_solns: manufactured_solutions, manufactured_electric_fields -using moment_kinetics.moment_kinetics_input: mk_input -using moment_kinetics.load_data: open_readonly_output_file, get_group, load_block_data, - load_coordinate_data, load_distributed_charged_pdf_slice, - load_distributed_neutral_pdf_slice, load_input, - load_mk_options, load_species_data, load_time_data +using moment_kinetics.load_data: close_run_info, get_run_info_no_setup, get_variable, + postproc_load_variable, em_variables, + ion_moment_variables, neutral_moment_variables, + all_moment_variables, ion_dfn_variables, + neutral_dfn_variables, all_dfn_variables, ion_variables, + neutral_variables, all_variables using moment_kinetics.initial_conditions: vpagrid_to_dzdt -using .shared_utils: calculate_and_write_frequencies, construct_global_zr_coords, - get_geometry_and_composition, read_distributed_zr_data! +using .shared_utils: calculate_and_write_frequencies, get_geometry_and_composition using moment_kinetics.type_definitions: mk_float, mk_int using moment_kinetics.velocity_moments: integrate_over_vspace, integrate_over_neutral_vspace using Combinatorics -using Glob using LaTeXStrings using LsqFit using MPI @@ -77,24 +76,6 @@ a TOML file. """ const input_dict_dfns = OrderedDict{String,Any}() -const em_variables = ("phi", "Er", "Ez") -const ion_moment_variables = ("density", "parallel_flow", "parallel_pressure", - "thermal_speed", "temperature", "parallel_heat_flux", - "collision_frequency", "sound_speed", "mach_number") -const neutral_moment_variables = ("density_neutral", "uz_neutral", "pz_neutral", - "thermal_speed_neutral", "temperature_neutral", - "qz_neutral") -const all_moment_variables = tuple(em_variables..., ion_moment_variables..., - neutral_moment_variables...) - -const ion_dfn_variables = ("f",) -const neutral_dfn_variables = ("f_neutral",) -const all_dfn_variables = tuple(ion_dfn_variables..., neutral_dfn_variables...) - -const ion_variables = tuple(ion_moment_variables..., ion_dfn_variables) -const neutral_variables = tuple(neutral_moment_variables..., neutral_dfn_variables) -const all_variables = tuple(all_moment_variables..., all_dfn_variables...) - const one_dimension_combinations_no_t = setdiff(all_dimensions, (:s, :sn)) const one_dimension_combinations = (:t, one_dimension_combinations_no_t...) const two_dimension_combinations_no_t = Tuple( @@ -439,7 +420,8 @@ Pass `input_file` to read the input from an input file other than Set up input, storing in the global [`input_dict`](@ref) and [`input_dict_dfns`](@ref) to be used in the various plotting and analysis functions. -The `run_info` that you are using (as returned by [`get_run_info`](@ref)) should be passed +The `run_info` that you are using (as returned by +[`moment_kinetics.load_data.get_run_info`](@ref)) should be passed to `run_info_moments` (if it contains only the moments), or `run_info_dfns` (if it also contains the distributions functions), or both (if you have loaded both sets of output). This allows default values to be set based on the grid sizes and number of time points @@ -757,222 +739,23 @@ can be passed to [`setup_makie_post_processing_input!`](@ref), to be used for de the remaining options. If either `itime_min` or `itime_max` are ≤0, their values are used as offsets from the final time index of the run. -By default `setup_makie_post_processing_input!()` is called at the end of -`get_run_info()`, for convenience when working interactively. Pass `do_setup=false` to -disable this. A post-processing input file can be passed to `setup_input_file` that will -be passed to `setup_makie_post_processing_input!()` if you do not want to use the default -input file. +`setup_makie_post_processing_input!()` is called at the end of `get_run_info()`, for +convenience when working interactively. Use +[moment_kinetics.load_data.get_run_info_no_setup](@ref) if you do not want this. A +post-processing input file can be passed to `setup_input_file` that will be passed to +`setup_makie_post_processing_input!()` if you do not want to use the default input file. """ -function get_run_info(run_dir::Union{AbstractString,Tuple{AbstractString,Union{Int,Nothing}}}...; - itime_min=1, itime_max=0, itime_skip=1, dfns=false, do_setup=true, - setup_input_file=nothing) - if length(run_dir) == 0 - error("No run_dir passed") - end - if length(run_dir) > 1 - run_info = Tuple(get_run_info(r; itime_min=itime_min, itime_max=itime_max, - itime_skip=itime_skip, dfns=dfns, do_setup=false) - for r ∈ run_dir) - if do_setup - if dfns - setup_makie_post_processing_input!( - setup_input_file; run_info_dfns=run_info, - allow_missing_input_file=(setup_input_file === nothing)) - else - setup_makie_post_processing_input!( - setup_input_file; run_info_moments=run_info, - allow_missing_input_file=(setup_input_file === nothing)) - end - end - return run_info - end - - this_run_dir = run_dir[1] - if isa(this_run_dir, Tuple) - if length(this_run_dir) != 2 - error("When a Tuple is passed for run_dir, expect it to have length 2. Got " - * "$this_run_dir") - end - this_run_dir, restart_index = this_run_dir - else - restart_index = nothing - end - - if !isa(this_run_dir, AbstractString) || !isa(restart_index, Union{Int,Nothing}) - error("Expected all `run_dir` arguments to be `String` or `(String, Int)` or " - * "`(String, Nothing)`. Got $run_dir") - end - - if !isdir(this_run_dir) - error("$this_run_dir is not a directory") - end - - # Normalise by removing any trailing slash - with a slash basename() would return an - # empty string - this_run_dir = rstrip(this_run_dir, '/') - - run_name = basename(this_run_dir) - base_prefix = joinpath(this_run_dir, run_name) - if restart_index === nothing - # Find output files from all restarts in the directory - counter = 1 - run_prefixes = Vector{String}() - while true - # Test if output files exist for this value of counter - prefix_with_count = base_prefix * "_$counter" - if length(glob(basename(prefix_with_count) * ".*.h5", dirname(prefix_with_count))) > 0 || - length(glob(basename(prefix_with_count) * ".*.cdf", dirname(prefix_with_count))) > 0 - - push!(run_prefixes, prefix_with_count) - else - # No more output files found - break - end - counter += 1 - end - # Add the final run which does not have a '_$counter' suffix - push!(run_prefixes, base_prefix) - run_prefixes = tuple(run_prefixes...) - elseif restart_index == -1 - run_prefixes = (base_prefix,) - elseif restart_index > 0 - run_prefixes = (base_prefix * "_$restart_index",) - else - error("Invalid restart_index=$restart_index") - end - - if dfns - ext = "dfns" - else - ext = "moments" - end - - has_data = all(length(glob(basename(p) * ".$ext*.h5", dirname(p))) > 0 || - length(glob(basename(p) * ".$ext*.cdf", dirname(p))) > 0 - for p ∈ run_prefixes) - if !has_data - println("No $ext data found for $run_prefixes, skipping $ext") - return nothing - end - - fids0 = Tuple(open_readonly_output_file(r, ext, printout=false) - for r ∈ run_prefixes) - nblocks = Tuple(load_block_data(f)[1] for f ∈ fids0) - if all(n == 1 for n ∈ nblocks) - # Did not use distributed memory, or used parallel_io - parallel_io = true - else - parallel_io = false - end - - nt_unskipped, time, restarts_nt = load_time_data(fids0) - if itime_min <= 0 - itime_min = nt_unskipped + itime_min - end - if itime_max <= 0 - itime_max = nt_unskipped + itime_max - end - time = time[itime_min:itime_skip:itime_max] - nt = length(time) - - # Get input and coordinates from the final restart - file_final_restart = fids0[end] - - input = load_input(file_final_restart) - - # obtain input options from moment_kinetics_input.jl - # and check input to catch errors - io_input, evolve_moments, t_input, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, - composition, species, collisions, geometry, drive_input, external_source_settings, - num_diss_params, manufactured_solns_input = mk_input(input) +function get_run_info(args...; do_setup=true, setup_input_file=nothing, dfns=false, + kwargs...) - n_ion_species, n_neutral_species = load_species_data(file_final_restart) - evolve_density, evolve_upar, evolve_ppar = load_mk_options(file_final_restart) - - z_local, z_local_spectral, z_chunk_size = - load_coordinate_data(file_final_restart, "z") - r_local, r_local_spectral, r_chunk_size = - load_coordinate_data(file_final_restart, "r") - r, r_spectral, z, z_spectral = construct_global_zr_coords(r_local, z_local) - - if dfns - vperp, vperp_spectral, vperp_chunk_size = - load_coordinate_data(file_final_restart, "vperp") - vpa, vpa_spectral, vpa_chunk_size = - load_coordinate_data(file_final_restart, "vpa") - - if n_neutral_species > 0 - vzeta, vzeta_spectral, vzeta_chunk_size = - load_coordinate_data(file_final_restart, "vzeta") - vr, vr_spectral, vr_chunk_size = - load_coordinate_data(file_final_restart, "vr") - vz, vz_spectral, vz_chunk_size = - load_coordinate_data(file_final_restart, "vz") - else - dummy_adv_input = advection_input("default", 1.0, 0.0, 0.0) - dummy_comm = MPI.COMM_NULL - dummy_input = grid_input("dummy", 1, 1, 1, 1, 0, 1.0, - "chebyshev_pseudospectral", "", "", "periodic", - dummy_adv_input, dummy_comm, "uniform") - vzeta, vzeta_spectral = define_coordinate(dummy_input) - vzeta_chunk_size = 1 - vr, vr_spectral = define_coordinate(dummy_input) - vr_chunk_size = 1 - vz, vz_spectral = define_coordinate(dummy_input) - vz_chunk_size = 1 - end - end - - if parallel_io - files = fids0 - else - # Don't keep open files as read_distributed_zr_data!(), etc. open the files - # themselves - files = run_prefixes - end + run_info = get_run_info_no_setup(args...; dfns=dfns, kwargs...) - if dfns - run_info = (run_name=run_name, run_prefix=base_prefix, parallel_io=parallel_io, - ext=ext, nblocks=nblocks, files=files, input=input, - n_ion_species=n_ion_species, n_neutral_species=n_neutral_species, - evolve_moments=evolve_moments, composition=composition, - species=species, collisions=collisions, geometry=geometry, - drive_input=drive_input, num_diss_params=num_diss_params, - evolve_density=evolve_density, evolve_upar=evolve_upar, - evolve_ppar=evolve_ppar, - manufactured_solns_input=manufactured_solns_input, nt=nt, - nt_unskipped=nt_unskipped, restarts_nt=restarts_nt, - itime_min=itime_min, itime_skip=itime_skip, itime_max=itime_max, - time=time, r=r, z=z, vperp=vperp, vpa=vpa, vzeta=vzeta, vr=vr, vz=vz, - r_local=r_local, z_local=z_local, r_spectral=r_spectral, - z_spectral=z_spectral, vperp_spectral=vperp_spectral, - vpa_spectral=vpa_spectral, vzeta_spectral=vzeta_spectral, - vr_spectral=vr_spectral, vz_spectral=vz_spectral, - r_chunk_size=r_chunk_size, z_chunk_size=z_chunk_size, - vperp_chunk_size=vperp_chunk_size, vpa_chunk_size=vpa_chunk_size, - vzeta_chunk_size=vzeta_chunk_size, vr_chunk_size=vr_chunk_size, - vz_chunk_size=vz_chunk_size, dfns=dfns) - if do_setup + if do_setup + if dfns setup_makie_post_processing_input!( setup_input_file; run_info_dfns=run_info, allow_missing_input_file=(setup_input_file === nothing)) - end - else - run_info = (run_name=run_name, run_prefix=base_prefix, parallel_io=parallel_io, - ext=ext, nblocks=nblocks, files=files, input=input, - n_ion_species=n_ion_species, n_neutral_species=n_neutral_species, - evolve_moments=evolve_moments, composition=composition, - species=species, collisions=collisions, geometry=geometry, - drive_input=drive_input, num_diss_params=num_diss_params, - evolve_density=evolve_density, evolve_upar=evolve_upar, - evolve_ppar=evolve_ppar, - manufactured_solns_input=manufactured_solns_input, nt=nt, - nt_unskipped=nt_unskipped, restarts_nt=restarts_nt, - itime_min=itime_min, itime_skip=itime_skip, itime_max=itime_max, - time=time, r=r, z=z, r_local=r_local, z_local=z_local, - r_spectral=r_spectral, z_spectral=z_spectral, - r_chunk_size=r_chunk_size, z_chunk_size=z_chunk_size, dfns=dfns) - if do_setup + else setup_makie_post_processing_input!( setup_input_file; run_info_moments=run_info, allow_missing_input_file=(setup_input_file === nothing)) @@ -982,392 +765,6 @@ function get_run_info(run_dir::Union{AbstractString,Tuple{AbstractString,Union{I return run_info end -""" - close_run_info(run_info) - -Close all the files in a run_info NamedTuple. -""" -function close_run_info(run_info) - if run_info === nothing - return nothing - end - if !run_info.parallel_io - # Files are not kept open, so nothing to do - return nothing - end - - for f ∈ run_info.files - close(f) - end - - return nothing -end - -""" - postproc_load_variable(run_info, variable_name; it=nothing, is=nothing, - ir=nothing, iz=nothing, ivperp=nothing, ivpa=nothing, - ivzeta=nothing, ivr=nothing, ivz=nothing) - -Load a variable - -`run_info` is the information about a run returned by [`get_run_info`](@ref). - -`variable_name` is the name of the variable to load. - -The keyword arguments `it`, `is`, `ir`, `iz`, `ivperp`, `ivpa`, `ivzeta`, `ivr`, and `ivz` -can be set to an integer or a range (e.g. `3:8` or `3:2:8`) to select subsets of the data. -Only the data for the subset requested will be loaded from the output file (mostly - when -loading fields or moments from runs which used `parallel_io = false`, the full array will -be loaded and then sliced). -""" -function postproc_load_variable(run_info, variable_name; it=nothing, is=nothing, - ir=nothing, iz=nothing, ivperp=nothing, ivpa=nothing, - ivzeta=nothing, ivr=nothing, ivz=nothing) - nt = run_info.nt - - if it === nothing - it = run_info.itime_min:run_info.itime_skip:run_info.itime_max - elseif isa(it, mk_int) - nt = 1 - it = collect(run_info.itime_min:run_info.itime_skip:run_info.itime_max)[it] - else - nt = length(it) - end - if is === nothing - # Can't use 'n_species' in a similar way to the way we treat other dims, because - # we don't know here if the variable is for ions or neutrals. - # Use Colon operator `:` when slice argument is `nothing` as when we pass that as - # an 'index', it selects the whole dimension. Brackets are needed around the `:` - # when assigning it to variables, etc. to avoid an error "LoadError: syntax: - # newline not allowed after ":" used for quoting". - is = (:) - elseif isa(is, mk_int) - nspecies = 1 - else - nspecies = length(is) - end - if ir === nothing - nr = run_info.r.n - ir = 1:nr - elseif isa(ir, mk_int) - nr = 1 - else - nr = length(ir) - end - if iz === nothing - nz = run_info.z.n - iz = 1:nz - elseif isa(iz, mk_int) - nz = 1 - else - nz = length(iz) - end - if ivperp === nothing - if :vperp ∈ keys(run_info) - # v-space coordinates only present if run_info contains distribution functions - nvperp = run_info.vperp.n - ivperp = 1:nvperp - else - nvperp = nothing - ivperp = nothing - end - elseif isa(ivperp, mk_int) - nvperp = 1 - else - nvperp = length(ivperp) - end - if ivpa === nothing - if :vpa ∈ keys(run_info) - # v-space coordinates only present if run_info contains distribution functions - nvpa = run_info.vpa.n - ivpa = 1:nvpa - else - nvpa = nothing - ivpa = nothing - end - elseif isa(ivpa, mk_int) - nvpa = 1 - else - nvpa = length(ivpa) - end - if ivzeta === nothing - if :vzeta ∈ keys(run_info) - # v-space coordinates only present if run_info contains distribution functions - nvzeta = run_info.vzeta.n - ivzeta = 1:nvzeta - else - nvzeta = nothing - ivzeta = nothing - end - elseif isa(ivzeta, mk_int) - nvzeta = 1 - else - nvzeta = length(ivzeta) - end - if ivr === nothing - if :vr ∈ keys(run_info) - # v-space coordinates only present if run_info contains distribution functions - nvr = run_info.vr.n - ivr = 1:nvr - else - nvr = nothing - ivr = nothing - end - elseif isa(ivr, mk_int) - nvr = 1 - else - nvr = length(ivr) - end - if ivz === nothing - if :vz ∈ keys(run_info) - # v-space coordinates only present if run_info contains distribution functions - nvz = run_info.vz.n - ivz = 1:nvz - else - nvz = nothing - ivz = nothing - end - elseif isa(ivz, mk_int) - nvz = 1 - else - nvz = length(ivz) - end - - if run_info.parallel_io - # Get HDF5/NetCDF variables directly and load slices - variable = Tuple(get_group(f, "dynamic_data")[variable_name] - for f ∈ run_info.files) - nd = ndims(variable[1]) - - if nd == 3 - # EM variable with dimensions (z,r,t) - dims = Vector{mk_int}() - !isa(iz, mk_int) && push!(dims, nz) - !isa(ir, mk_int) && push!(dims, nr) - !isa(it, mk_int) && push!(dims, nt) - result = allocate_float(dims...) - elseif nd == 4 - # moment variable with dimensions (z,r,s,t) - # Get nspecies from the variable, not from run_info, because it might be - # either ion or neutral - dims = Vector{mk_int}() - !isa(iz, mk_int) && push!(dims, nz) - !isa(ir, mk_int) && push!(dims, nr) - if is === (:) - nspecies = size(variable[1], 3) - push!(dims, nspecies) - elseif !isa(is, mk_int) - push!(dims, nspecies) - end - !isa(it, mk_int) && push!(dims, nt) - result = allocate_float(dims...) - elseif nd == 6 - # ion distribution function variable with dimensions (vpa,vperp,z,r,s,t) - nspecies = size(variable[1], 5) - dims = Vector{mk_int}() - !isa(ivpa, mk_int) && push!(dims, nvpa) - !isa(ivperp, mk_int) && push!(dims, nvperp) - !isa(iz, mk_int) && push!(dims, nz) - !isa(ir, mk_int) && push!(dims, nr) - if is === (:) - nspecies = size(variable[1], 3) - push!(dims, nspecies) - elseif !isa(is, mk_int) - push!(dims, nspecies) - end - !isa(it, mk_int) && push!(dims, nt) - result = allocate_float(dims...) - elseif nd == 7 - # neutral distribution function variable with dimensions (vz,vr,vzeta,z,r,s,t) - nspecies = size(variable[1], 6) - dims = Vector{mk_int}() - !isa(ivz, mk_int) && push!(dims, nvz) - !isa(ivr, mk_int) && push!(dims, nvr) - !isa(ivzeta, mk_int) && push!(dims, nvzeta) - !isa(iz, mk_int) && push!(dims, nz) - !isa(ir, mk_int) && push!(dims, nr) - if is === (:) - nspecies = size(variable[1], 3) - push!(dims, nspecies) - elseif !isa(is, mk_int) - push!(dims, nspecies) - end - !isa(it, mk_int) && push!(dims, nt) - result = allocate_float(dims...) - else - error("Unsupported number of dimensions ($nd) for '$variable_name'.") - end - - local_it_start = 1 - global_it_start = 1 - for v ∈ variable - # For restarts, the first time point is a duplicate of the last time - # point of the previous restart. Use `offset` to skip this point. - offset = local_it_start == 1 ? 0 : 1 - local_nt = size(v, nd) - offset - local_it_end = local_it_start+local_nt-1 - - if isa(it, mk_int) - tind = it - local_it_start + 1 - if tind < 1 - error("Trying to select time index before the beginning of this " - * "restart, should have finished already") - elseif tind <= local_nt - # tind is within this restart's time range, so get result - if nd == 3 - result .= v[iz,ir,tind] - elseif nd == 4 - result .= v[iz,ir,is,tind] - elseif nd == 6 - result .= v[ivpa,ivperp,iz,ir,is,tind] - elseif nd == 7 - result .= v[ivz,ivr,ivzeta,iz,ir,is,tind] - else - error("Unsupported combination nd=$nd, ir=$ir, iz=$iz, ivperp=$ivperp " - * "ivpa=$ivpa, ivzeta=$ivzeta, ivr=$ivr, ivz=$ivz.") - end - - # Already got the data for `it`, so end loop - break - end - else - tinds = collect(i - local_it_start + 1 + offset for i ∈ it - if local_it_start <= i <= local_it_end) - # Convert tinds to slice, as we know the spacing is constant - if length(tinds) != 0 - # There is some data in this file - if length(tinds) > 1 - tstep = tinds[2] - tinds[begin] - else - tstep = 1 - end - tinds = tinds[begin]:tstep:tinds[end] - global_it_end = global_it_start + length(tinds) - 1 - - if nd == 3 - selectdim(result, ndims(result), global_it_start:global_it_end) .= v[iz,ir,tinds] - elseif nd == 4 - selectdim(result, ndims(result), global_it_start:global_it_end) .= v[iz,ir,is,tinds] - elseif nd == 6 - selectdim(result, ndims(result), global_it_start:global_it_end) .= v[ivpa,ivperp,iz,ir,is,tinds] - elseif nd == 7 - selectdim(result, ndims(result), global_it_start:global_it_end) .= v[ivz,ivr,ivzeta,iz,ir,is,tinds] - else - error("Unsupported combination nd=$nd, ir=$ir, iz=$iz, ivperp=$ivperp " - * "ivpa=$ivpa, ivzeta=$ivzeta, ivr=$ivr, ivz=$ivz.") - end - - global_it_start = global_it_end + 1 - end - end - - local_it_start = local_it_end + 1 - end - else - # Use existing distributed I/O loading functions - if variable_name ∈ em_variables - nd = 3 - elseif variable_name ∈ ion_dfn_variables - nd = 6 - elseif variable_name ∈ neutral_dfn_variables - nd = 7 - else - # Ion or neutral moment variable - nd = 4 - end - - if nd == 3 - result = allocate_float(run_info.z.n, run_info.r.n, run_info.nt) - read_distributed_zr_data!(result, variable_name, run_info.files, - run_info.ext, run_info.nblocks, run_info.z_local.n, - run_info.r_local.n, run_info.itime_skip) - result = result[iz,ir,it] - elseif nd == 4 - # If we ever have neutrals included but n_neutral_species != n_ion_species, - # then this will fail - in that case would need some way to specify that we - # need to read a neutral moment variable rather than an ion moment variable - # here. - result = allocate_float(run_info.z.n, run_info.r.n, run_info.n_ion_species, - run_info.nt) - read_distributed_zr_data!(result, variable_name, run_info.files, - run_info.ext, run_info.nblocks, run_info.z_local.n, - run_info.r_local.n, run_info.itime_skip) - result = result[iz,ir,is,it] - elseif nd === 6 - result = load_distributed_charged_pdf_slice(run_info.files, run_info.nblocks, - it, run_info.n_ion_species, - run_info.r_local, - run_info.z_local, run_info.vperp, - run_info.vpa; - is=(is === (:) ? nothing : is), - ir=ir, iz=iz, ivperp=ivperp, - ivpa=ivpa) - elseif nd === 7 - result = load_distributed_neutral_pdf_slice(run_info.files, run_info.nblocks, - it, run_info.n_ion_species, - run_info.r_local, - run_info.z_local, run_info.vzeta, - run_info.vr, run_info.vz; - isn=(is === (:) ? nothing : is), - ir=ir, iz=iz, ivzeta=ivzeta, - ivr=ivr, ivz=ivz) - end - end - - return result -end - -""" - get_variable(run_info::Tuple, variable_name; kwargs...) - get_variable(run_info, variable_name; kwargs...) - -Get an array (or Tuple of arrays, if `run_info` is a Tuple) of the data for -`variable_name` from `run_info`. - -Some derived variables need to be calculated from the saved output, not just loaded from -file (with `postproc_load_variable`). This function takes care of that calculation, and -handles the case where `run_info` is a Tuple (which `postproc_load_data` does not handle). - -`kwargs...` are passed through to `postproc_load_variable()`. -""" -function get_variable end - -function get_variable(run_info::Tuple, variable_name; kwargs...) - return Tuple(get_variable(ri, variable_name; kwargs...) for ri ∈ run_info) -end - -function get_variable(run_info, variable_name; kwargs...) - if variable_name == "temperature" - vth = postproc_load_variable(run_info, "thermal_speed") - variable = vth.^2 - elseif variable_name == "collision_frequency" - n = postproc_load_variable(run_info, "density") - vth = postproc_load_variable(run_info, "thermal_speed") - variable = get_collision_frequency(run_info.collisions, n, vth) - elseif variable_name == "temperature_neutral" - vth = postproc_load_variable(run_info, "thermal_speed_neutral") - variable = vth.^2 - elseif variable_name == "sound_speed" - T_e = run_info.composition.T_e - T_i = get_variable(run_info, "temperature"; kwargs...) - - # Adiabatic index. Not too clear what value should be (see e.g. [Riemann 1991, - # below eq. (39)], or discussion of Bohm criterion in Stangeby's book. - gamma = 3.0 - - # Factor of 0.5 needed because temperatures are normalised to mi*cref^2, not Tref - variable = @. sqrt(0.5*(T_e + gamma*T_i)) - elseif variable_name == "mach_number" - upar = get_variable(run_info, "parallel_flow"; kwargs...) - cs = get_variable(run_info, "sound_speed"; kwargs...) - variable = upar ./ cs - else - variable = postproc_load_variable(run_info, variable_name) - end - - return variable -end - const chunk_size_1d = 10000 const chunk_size_2d = 100 struct VariableCache{T1,T2,N} diff --git a/makie_post_processing/src/shared_utils.jl b/makie_post_processing/src/shared_utils.jl index 700a24b24..001082a67 100644 --- a/makie_post_processing/src/shared_utils.jl +++ b/makie_post_processing/src/shared_utils.jl @@ -1,11 +1,13 @@ module shared_utils -export calculate_and_write_frequencies, construct_global_zr_coords, - get_geometry_and_composition, read_distributed_zr_data! +export calculate_and_write_frequencies, get_geometry_and_composition using moment_kinetics.analysis: fit_delta_phi_mode +using moment_kinetics.array_allocation: allocate_float using moment_kinetics.coordinates: define_coordinate -using moment_kinetics.input_structs: grid_input +using moment_kinetics.input_structs: boltzmann_electron_response, + boltzmann_electron_response_with_simple_sheath, + grid_input, geometry_input, species_composition using moment_kinetics.type_definitions: mk_float, mk_int using MPI @@ -57,23 +59,6 @@ function calculate_and_write_frequencies(run_name, ntime, time, z, itime_min, it return frequency, growth_rate, shifted_time, fitted_delta_phi end -""" -""" -function construct_global_zr_coords(r_local, z_local) - - function make_global_input(coord_local) - return grid_input(coord_local.name, coord_local.ngrid, - coord_local.nelement_global, coord_local.nelement_global, 1, 0, coord_local.L, - coord_local.discretization, coord_local.fd_option, coord_local.cheb_option, coord_local.bc, - coord_local.advection, MPI.COMM_NULL, coord_local.element_spacing_option) - end - - r_global, r_global_spectral = define_coordinate(make_global_input(r_local)) - z_global, z_global_spectral = define_coordinate(make_global_input(z_local)) - - return r_global, r_global_spectral, z_global, z_global_spectral -end - """ """ function get_geometry_and_composition(scan_input,n_ion_species,n_neutral_species) @@ -130,75 +115,4 @@ function get_geometry_and_composition(scan_input,n_ion_species,n_neutral_species end -""" -Read data which is a function of (z,r,t) or (z,r,species,t) - -run_names is a tuple. If it has more than one entry, this means that there are multiple -restarts (which are sequential in time), so concatenate the data from each entry together. -""" -function read_distributed_zr_data!(var::Array{mk_float,N}, var_name::String, - run_names::Tuple, file_key::String, nblocks::Tuple, - nz_local::mk_int,nr_local::mk_int,iskip::mk_int) where N - # dimension of var is [z,r,species,t] - - local_tind_start = 1 - local_tind_end = -1 - global_tind_start = 1 - global_tind_end = -1 - for (run_name, nb) in zip(run_names, nblocks) - for iblock in 0:nb-1 - fid = open_readonly_output_file(run_name,file_key,iblock=iblock,printout=false) - group = get_group(fid, "dynamic_data") - var_local = load_variable(group, var_name) - - ntime_local = size(var_local, N) - - # offset is the amount we have to skip at the beginning of this restart to - # line up properly with having outputs every iskip since the beginning of the - # first restart. - # Note: use rem(x,y,RoundDown) here because this gives a result that's - # definitely between 0 and y, whereas rem(x,y) or mod(x,y) give negative - # results for negative x. - offset = rem(1 - (local_tind_start-1), iskip, RoundDown) - if offset == 0 - # Actually want offset in the range [1,iskip], so correct if rem() - # returned 0 - offset = iskip - end - if local_tind_start > 1 - # The run being loaded is a restart (as local_tind_start=1 for the first - # run), so skip the first point, as this is a duplicate of the last point - # of the previous restart - offset += 1 - end - - local_tind_end = local_tind_start + ntime_local - 1 - global_tind_end = global_tind_start + length(offset:iskip:ntime_local) - 1 - - z_irank, z_nrank, r_irank, r_nrank = load_rank_data(fid) - - # min index set to avoid double assignment of repeated points - # 1 if irank = 0, 2 otherwise - imin_r = min(1,r_irank) + 1 - imin_z = min(1,z_irank) + 1 - for ir_local in imin_r:nr_local - for iz_local in imin_z:nz_local - ir_global = iglobal_func(ir_local,r_irank,nr_local) - iz_global = iglobal_func(iz_local,z_irank,nz_local) - if N == 4 - var[iz_global,ir_global,:,global_tind_start:global_tind_end] .= var_local[iz_local,ir_local,:,offset:iskip:end] - elseif N == 3 - var[iz_global,ir_global,global_tind_start:global_tind_end] .= var_local[iz_local,ir_local,offset:iskip:end] - else - error("Unsupported number of dimensions: $N") - end - end - end - close(fid) - end - local_tind_start = local_tind_end + 1 - global_tind_start = global_tind_end + 1 - end -end - end # shared_utils.jl diff --git a/moment_kinetics/src/load_data.jl b/moment_kinetics/src/load_data.jl index 567d4949b..a312d2d56 100644 --- a/moment_kinetics/src/load_data.jl +++ b/moment_kinetics/src/load_data.jl @@ -13,6 +13,7 @@ export load_time_data export load_block_data export load_rank_data export load_species_data +export read_distributed_zr_data! using ..array_allocation: allocate_float using ..coordinates: coordinate, define_coordinate @@ -20,11 +21,31 @@ using ..file_io: check_io_implementation, get_group, get_subgroup_keys, get_vari using ..input_structs: advection_input, grid_input, hdf5, netcdf using ..interpolation: interpolate_to_grid_1d! using ..looping -using ..type_definitions: mk_int +using ..moment_kinetics_input: mk_input +using ..type_definitions: mk_float, mk_int +using Glob using HDF5 using MPI +const em_variables = ("phi", "Er", "Ez") +const ion_moment_variables = ("density", "parallel_flow", "parallel_pressure", + "thermal_speed", "temperature", "parallel_heat_flux", + "collision_frequency", "sound_speed", "mach_number") +const neutral_moment_variables = ("density_neutral", "uz_neutral", "pz_neutral", + "thermal_speed_neutral", "temperature_neutral", + "qz_neutral") +const all_moment_variables = tuple(em_variables..., ion_moment_variables..., + neutral_moment_variables...) + +const ion_dfn_variables = ("f",) +const neutral_dfn_variables = ("f_neutral",) +const all_dfn_variables = tuple(ion_dfn_variables..., neutral_dfn_variables...) + +const ion_variables = tuple(ion_moment_variables..., ion_dfn_variables) +const neutral_variables = tuple(neutral_moment_variables..., neutral_dfn_variables) +const all_variables = tuple(all_moment_variables..., all_dfn_variables...) + function open_file_to_read end function open_file_to_read(::Val{hdf5}, filename) return h5open(filename, "r") @@ -2260,4 +2281,707 @@ function ilocal_func(iglobal,irank,nlocal) return iglobal - irank*(nlocal - 1) end +""" + get_run_info_no_setup(run_dir...; itime_min=1, itime_max=0, itime_skip=1, dfns=false, + do_setup=true, setup_input_file=nothing) + get_run_info_no_setup((run_dir, restart_index)...; itime_min=1, itime_max=0, + itime_skip=1, dfns=false, do_setup=true, setup_input_file=nothing) + +Get file handles and other info for a single run + +`run_dir` is the directory to read output from. + +`restart_index` can be given by passing a Tuple, e.g. `("runs/example", 42)` as the +positional argument. It specifies which restart to read if there are multiple restarts. If +no `restart_index` is given or if `nothing` is passed, read all restarts and concatenate +them. An integer value reads the restart with that index - `-1` indicates the latest +restart (which does not have an index). + +Several runs can be loaded at the same time by passing multiple positional arguments. Each +argument can be a String `run_dir` giving a directory to read output from or a Tuple +`(run_dir, restart_index)` giving both a directory and a restart index (it is allowed to +mix Strings and Tuples in a call). + +By default load data from moments files, pass `dfns=true` to load from distribution +functions files. + +The `itime_min`, `itime_max` and `itime_skip` options can be used to select only a slice +of time points when loading data. In `makie_post_process` these options are read from the +input (if they are set) before `get_run_info()` is called, so that the `run_info` returned +can be passed to [`setup_makie_post_processing_input!`](@ref), to be used for defaults for +the remaining options. If either `itime_min` or `itime_max` are ≤0, their values are used +as offsets from the final time index of the run. +""" +function get_run_info_no_setup(run_dir::Union{AbstractString,Tuple{AbstractString,Union{Int,Nothing}}}...; + itime_min=1, itime_max=0, itime_skip=1, dfns=false) + if length(run_dir) == 0 + error("No run_dir passed") + end + if length(run_dir) > 1 + run_info = Tuple(get_run_info_no_setup(r; itime_min=itime_min, + itime_max=itime_max, itime_skip=itime_skip, + dfns=dfns) + for r ∈ run_dir) + return run_info + end + + this_run_dir = run_dir[1] + if isa(this_run_dir, Tuple) + if length(this_run_dir) != 2 + error("When a Tuple is passed for run_dir, expect it to have length 2. Got " + * "$this_run_dir") + end + this_run_dir, restart_index = this_run_dir + else + restart_index = nothing + end + + if !isa(this_run_dir, AbstractString) || !isa(restart_index, Union{Int,Nothing}) + error("Expected all `run_dir` arguments to be `String` or `(String, Int)` or " + * "`(String, Nothing)`. Got $run_dir") + end + + if !isdir(this_run_dir) + error("$this_run_dir is not a directory") + end + + # Normalise by removing any trailing slash - with a slash basename() would return an + # empty string + this_run_dir = rstrip(this_run_dir, '/') + + run_name = basename(this_run_dir) + base_prefix = joinpath(this_run_dir, run_name) + if restart_index === nothing + # Find output files from all restarts in the directory + counter = 1 + run_prefixes = Vector{String}() + while true + # Test if output files exist for this value of counter + prefix_with_count = base_prefix * "_$counter" + if length(glob(basename(prefix_with_count) * ".*.h5", dirname(prefix_with_count))) > 0 || + length(glob(basename(prefix_with_count) * ".*.cdf", dirname(prefix_with_count))) > 0 + + push!(run_prefixes, prefix_with_count) + else + # No more output files found + break + end + counter += 1 + end + # Add the final run which does not have a '_$counter' suffix + push!(run_prefixes, base_prefix) + run_prefixes = tuple(run_prefixes...) + elseif restart_index == -1 + run_prefixes = (base_prefix,) + elseif restart_index > 0 + run_prefixes = (base_prefix * "_$restart_index",) + else + error("Invalid restart_index=$restart_index") + end + + if dfns + ext = "dfns" + else + ext = "moments" + end + + has_data = all(length(glob(basename(p) * ".$ext*.h5", dirname(p))) > 0 || + length(glob(basename(p) * ".$ext*.cdf", dirname(p))) > 0 + for p ∈ run_prefixes) + if !has_data + println("No $ext data found for $run_prefixes, skipping $ext") + return nothing + end + + fids0 = Tuple(open_readonly_output_file(r, ext, printout=false) + for r ∈ run_prefixes) + nblocks = Tuple(load_block_data(f)[1] for f ∈ fids0) + if all(n == 1 for n ∈ nblocks) + # Did not use distributed memory, or used parallel_io + parallel_io = true + else + parallel_io = false + end + + nt_unskipped, time, restarts_nt = load_time_data(fids0) + if itime_min <= 0 + itime_min = nt_unskipped + itime_min + end + if itime_max <= 0 + itime_max = nt_unskipped + itime_max + end + time = time[itime_min:itime_skip:itime_max] + nt = length(time) + + # Get input and coordinates from the final restart + file_final_restart = fids0[end] + + input = load_input(file_final_restart) + + # obtain input options from moment_kinetics_input.jl + # and check input to catch errors + io_input, evolve_moments, t_input, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, + composition, species, collisions, geometry, drive_input, external_source_settings, + num_diss_params, manufactured_solns_input = mk_input(input) + + n_ion_species, n_neutral_species = load_species_data(file_final_restart) + evolve_density, evolve_upar, evolve_ppar = load_mk_options(file_final_restart) + + z_local, z_local_spectral, z_chunk_size = + load_coordinate_data(file_final_restart, "z") + r_local, r_local_spectral, r_chunk_size = + load_coordinate_data(file_final_restart, "r") + r, r_spectral, z, z_spectral = construct_global_zr_coords(r_local, z_local) + + if dfns + vperp, vperp_spectral, vperp_chunk_size = + load_coordinate_data(file_final_restart, "vperp") + vpa, vpa_spectral, vpa_chunk_size = + load_coordinate_data(file_final_restart, "vpa") + + if n_neutral_species > 0 + vzeta, vzeta_spectral, vzeta_chunk_size = + load_coordinate_data(file_final_restart, "vzeta") + vr, vr_spectral, vr_chunk_size = + load_coordinate_data(file_final_restart, "vr") + vz, vz_spectral, vz_chunk_size = + load_coordinate_data(file_final_restart, "vz") + else + dummy_adv_input = advection_input("default", 1.0, 0.0, 0.0) + dummy_comm = MPI.COMM_NULL + dummy_input = grid_input("dummy", 1, 1, 1, 1, 0, 1.0, + "chebyshev_pseudospectral", "", "", "periodic", + dummy_adv_input, dummy_comm, "uniform") + vzeta, vzeta_spectral = define_coordinate(dummy_input) + vzeta_chunk_size = 1 + vr, vr_spectral = define_coordinate(dummy_input) + vr_chunk_size = 1 + vz, vz_spectral = define_coordinate(dummy_input) + vz_chunk_size = 1 + end + end + + if parallel_io + files = fids0 + else + # Don't keep open files as read_distributed_zr_data!(), etc. open the files + # themselves + files = run_prefixes + end + + if dfns + run_info = (run_name=run_name, run_prefix=base_prefix, parallel_io=parallel_io, + ext=ext, nblocks=nblocks, files=files, input=input, + n_ion_species=n_ion_species, n_neutral_species=n_neutral_species, + evolve_moments=evolve_moments, composition=composition, + species=species, collisions=collisions, geometry=geometry, + drive_input=drive_input, num_diss_params=num_diss_params, + evolve_density=evolve_density, evolve_upar=evolve_upar, + evolve_ppar=evolve_ppar, + manufactured_solns_input=manufactured_solns_input, nt=nt, + nt_unskipped=nt_unskipped, restarts_nt=restarts_nt, + itime_min=itime_min, itime_skip=itime_skip, itime_max=itime_max, + time=time, r=r, z=z, vperp=vperp, vpa=vpa, vzeta=vzeta, vr=vr, vz=vz, + r_local=r_local, z_local=z_local, r_spectral=r_spectral, + z_spectral=z_spectral, vperp_spectral=vperp_spectral, + vpa_spectral=vpa_spectral, vzeta_spectral=vzeta_spectral, + vr_spectral=vr_spectral, vz_spectral=vz_spectral, + r_chunk_size=r_chunk_size, z_chunk_size=z_chunk_size, + vperp_chunk_size=vperp_chunk_size, vpa_chunk_size=vpa_chunk_size, + vzeta_chunk_size=vzeta_chunk_size, vr_chunk_size=vr_chunk_size, + vz_chunk_size=vz_chunk_size, dfns=dfns) + else + run_info = (run_name=run_name, run_prefix=base_prefix, parallel_io=parallel_io, + ext=ext, nblocks=nblocks, files=files, input=input, + n_ion_species=n_ion_species, n_neutral_species=n_neutral_species, + evolve_moments=evolve_moments, composition=composition, + species=species, collisions=collisions, geometry=geometry, + drive_input=drive_input, num_diss_params=num_diss_params, + evolve_density=evolve_density, evolve_upar=evolve_upar, + evolve_ppar=evolve_ppar, + manufactured_solns_input=manufactured_solns_input, nt=nt, + nt_unskipped=nt_unskipped, restarts_nt=restarts_nt, + itime_min=itime_min, itime_skip=itime_skip, itime_max=itime_max, + time=time, r=r, z=z, r_local=r_local, z_local=z_local, + r_spectral=r_spectral, z_spectral=z_spectral, + r_chunk_size=r_chunk_size, z_chunk_size=z_chunk_size, dfns=dfns) + end + + return run_info +end + +""" + close_run_info(run_info) + +Close all the files in a run_info NamedTuple. +""" +function close_run_info(run_info) + if run_info === nothing + return nothing + end + if !run_info.parallel_io + # Files are not kept open, so nothing to do + return nothing + end + + for f ∈ run_info.files + close(f) + end + + return nothing +end + +""" + postproc_load_variable(run_info, variable_name; it=nothing, is=nothing, + ir=nothing, iz=nothing, ivperp=nothing, ivpa=nothing, + ivzeta=nothing, ivr=nothing, ivz=nothing) + +Load a variable + +`run_info` is the information about a run returned by [`get_run_info`](@ref). + +`variable_name` is the name of the variable to load. + +The keyword arguments `it`, `is`, `ir`, `iz`, `ivperp`, `ivpa`, `ivzeta`, `ivr`, and `ivz` +can be set to an integer or a range (e.g. `3:8` or `3:2:8`) to select subsets of the data. +Only the data for the subset requested will be loaded from the output file (mostly - when +loading fields or moments from runs which used `parallel_io = false`, the full array will +be loaded and then sliced). +""" +function postproc_load_variable(run_info, variable_name; it=nothing, is=nothing, + ir=nothing, iz=nothing, ivperp=nothing, ivpa=nothing, + ivzeta=nothing, ivr=nothing, ivz=nothing) + nt = run_info.nt + + if it === nothing + it = run_info.itime_min:run_info.itime_skip:run_info.itime_max + elseif isa(it, mk_int) + nt = 1 + it = collect(run_info.itime_min:run_info.itime_skip:run_info.itime_max)[it] + else + nt = length(it) + end + if is === nothing + # Can't use 'n_species' in a similar way to the way we treat other dims, because + # we don't know here if the variable is for ions or neutrals. + # Use Colon operator `:` when slice argument is `nothing` as when we pass that as + # an 'index', it selects the whole dimension. Brackets are needed around the `:` + # when assigning it to variables, etc. to avoid an error "LoadError: syntax: + # newline not allowed after ":" used for quoting". + is = (:) + elseif isa(is, mk_int) + nspecies = 1 + else + nspecies = length(is) + end + if ir === nothing + nr = run_info.r.n + ir = 1:nr + elseif isa(ir, mk_int) + nr = 1 + else + nr = length(ir) + end + if iz === nothing + nz = run_info.z.n + iz = 1:nz + elseif isa(iz, mk_int) + nz = 1 + else + nz = length(iz) + end + if ivperp === nothing + if :vperp ∈ keys(run_info) + # v-space coordinates only present if run_info contains distribution functions + nvperp = run_info.vperp.n + ivperp = 1:nvperp + else + nvperp = nothing + ivperp = nothing + end + elseif isa(ivperp, mk_int) + nvperp = 1 + else + nvperp = length(ivperp) + end + if ivpa === nothing + if :vpa ∈ keys(run_info) + # v-space coordinates only present if run_info contains distribution functions + nvpa = run_info.vpa.n + ivpa = 1:nvpa + else + nvpa = nothing + ivpa = nothing + end + elseif isa(ivpa, mk_int) + nvpa = 1 + else + nvpa = length(ivpa) + end + if ivzeta === nothing + if :vzeta ∈ keys(run_info) + # v-space coordinates only present if run_info contains distribution functions + nvzeta = run_info.vzeta.n + ivzeta = 1:nvzeta + else + nvzeta = nothing + ivzeta = nothing + end + elseif isa(ivzeta, mk_int) + nvzeta = 1 + else + nvzeta = length(ivzeta) + end + if ivr === nothing + if :vr ∈ keys(run_info) + # v-space coordinates only present if run_info contains distribution functions + nvr = run_info.vr.n + ivr = 1:nvr + else + nvr = nothing + ivr = nothing + end + elseif isa(ivr, mk_int) + nvr = 1 + else + nvr = length(ivr) + end + if ivz === nothing + if :vz ∈ keys(run_info) + # v-space coordinates only present if run_info contains distribution functions + nvz = run_info.vz.n + ivz = 1:nvz + else + nvz = nothing + ivz = nothing + end + elseif isa(ivz, mk_int) + nvz = 1 + else + nvz = length(ivz) + end + + if run_info.parallel_io + # Get HDF5/NetCDF variables directly and load slices + variable = Tuple(get_group(f, "dynamic_data")[variable_name] + for f ∈ run_info.files) + nd = ndims(variable[1]) + + if nd == 3 + # EM variable with dimensions (z,r,t) + dims = Vector{mk_int}() + !isa(iz, mk_int) && push!(dims, nz) + !isa(ir, mk_int) && push!(dims, nr) + !isa(it, mk_int) && push!(dims, nt) + result = allocate_float(dims...) + elseif nd == 4 + # moment variable with dimensions (z,r,s,t) + # Get nspecies from the variable, not from run_info, because it might be + # either ion or neutral + dims = Vector{mk_int}() + !isa(iz, mk_int) && push!(dims, nz) + !isa(ir, mk_int) && push!(dims, nr) + if is === (:) + nspecies = size(variable[1], 3) + push!(dims, nspecies) + elseif !isa(is, mk_int) + push!(dims, nspecies) + end + !isa(it, mk_int) && push!(dims, nt) + result = allocate_float(dims...) + elseif nd == 6 + # ion distribution function variable with dimensions (vpa,vperp,z,r,s,t) + nspecies = size(variable[1], 5) + dims = Vector{mk_int}() + !isa(ivpa, mk_int) && push!(dims, nvpa) + !isa(ivperp, mk_int) && push!(dims, nvperp) + !isa(iz, mk_int) && push!(dims, nz) + !isa(ir, mk_int) && push!(dims, nr) + if is === (:) + nspecies = size(variable[1], 3) + push!(dims, nspecies) + elseif !isa(is, mk_int) + push!(dims, nspecies) + end + !isa(it, mk_int) && push!(dims, nt) + result = allocate_float(dims...) + elseif nd == 7 + # neutral distribution function variable with dimensions (vz,vr,vzeta,z,r,s,t) + nspecies = size(variable[1], 6) + dims = Vector{mk_int}() + !isa(ivz, mk_int) && push!(dims, nvz) + !isa(ivr, mk_int) && push!(dims, nvr) + !isa(ivzeta, mk_int) && push!(dims, nvzeta) + !isa(iz, mk_int) && push!(dims, nz) + !isa(ir, mk_int) && push!(dims, nr) + if is === (:) + nspecies = size(variable[1], 3) + push!(dims, nspecies) + elseif !isa(is, mk_int) + push!(dims, nspecies) + end + !isa(it, mk_int) && push!(dims, nt) + result = allocate_float(dims...) + else + error("Unsupported number of dimensions ($nd) for '$variable_name'.") + end + + local_it_start = 1 + global_it_start = 1 + for v ∈ variable + # For restarts, the first time point is a duplicate of the last time + # point of the previous restart. Use `offset` to skip this point. + offset = local_it_start == 1 ? 0 : 1 + local_nt = size(v, nd) - offset + local_it_end = local_it_start+local_nt-1 + + if isa(it, mk_int) + tind = it - local_it_start + 1 + if tind < 1 + error("Trying to select time index before the beginning of this " + * "restart, should have finished already") + elseif tind <= local_nt + # tind is within this restart's time range, so get result + if nd == 3 + result .= v[iz,ir,tind] + elseif nd == 4 + result .= v[iz,ir,is,tind] + elseif nd == 6 + result .= v[ivpa,ivperp,iz,ir,is,tind] + elseif nd == 7 + result .= v[ivz,ivr,ivzeta,iz,ir,is,tind] + else + error("Unsupported combination nd=$nd, ir=$ir, iz=$iz, ivperp=$ivperp " + * "ivpa=$ivpa, ivzeta=$ivzeta, ivr=$ivr, ivz=$ivz.") + end + + # Already got the data for `it`, so end loop + break + end + else + tinds = collect(i - local_it_start + 1 + offset for i ∈ it + if local_it_start <= i <= local_it_end) + # Convert tinds to slice, as we know the spacing is constant + if length(tinds) != 0 + # There is some data in this file + if length(tinds) > 1 + tstep = tinds[2] - tinds[begin] + else + tstep = 1 + end + tinds = tinds[begin]:tstep:tinds[end] + global_it_end = global_it_start + length(tinds) - 1 + + if nd == 3 + selectdim(result, ndims(result), global_it_start:global_it_end) .= v[iz,ir,tinds] + elseif nd == 4 + selectdim(result, ndims(result), global_it_start:global_it_end) .= v[iz,ir,is,tinds] + elseif nd == 6 + selectdim(result, ndims(result), global_it_start:global_it_end) .= v[ivpa,ivperp,iz,ir,is,tinds] + elseif nd == 7 + selectdim(result, ndims(result), global_it_start:global_it_end) .= v[ivz,ivr,ivzeta,iz,ir,is,tinds] + else + error("Unsupported combination nd=$nd, ir=$ir, iz=$iz, ivperp=$ivperp " + * "ivpa=$ivpa, ivzeta=$ivzeta, ivr=$ivr, ivz=$ivz.") + end + + global_it_start = global_it_end + 1 + end + end + + local_it_start = local_it_end + 1 + end + else + # Use existing distributed I/O loading functions + if variable_name ∈ em_variables + nd = 3 + elseif variable_name ∈ ion_dfn_variables + nd = 6 + elseif variable_name ∈ neutral_dfn_variables + nd = 7 + else + # Ion or neutral moment variable + nd = 4 + end + + if nd == 3 + result = allocate_float(run_info.z.n, run_info.r.n, run_info.nt) + read_distributed_zr_data!(result, variable_name, run_info.files, + run_info.ext, run_info.nblocks, run_info.z_local.n, + run_info.r_local.n, run_info.itime_skip) + result = result[iz,ir,it] + elseif nd == 4 + # If we ever have neutrals included but n_neutral_species != n_ion_species, + # then this will fail - in that case would need some way to specify that we + # need to read a neutral moment variable rather than an ion moment variable + # here. + result = allocate_float(run_info.z.n, run_info.r.n, run_info.n_ion_species, + run_info.nt) + read_distributed_zr_data!(result, variable_name, run_info.files, + run_info.ext, run_info.nblocks, run_info.z_local.n, + run_info.r_local.n, run_info.itime_skip) + result = result[iz,ir,is,it] + elseif nd === 6 + result = load_distributed_charged_pdf_slice(run_info.files, run_info.nblocks, + it, run_info.n_ion_species, + run_info.r_local, + run_info.z_local, run_info.vperp, + run_info.vpa; + is=(is === (:) ? nothing : is), + ir=ir, iz=iz, ivperp=ivperp, + ivpa=ivpa) + elseif nd === 7 + result = load_distributed_neutral_pdf_slice(run_info.files, run_info.nblocks, + it, run_info.n_ion_species, + run_info.r_local, + run_info.z_local, run_info.vzeta, + run_info.vr, run_info.vz; + isn=(is === (:) ? nothing : is), + ir=ir, iz=iz, ivzeta=ivzeta, + ivr=ivr, ivz=ivz) + end + end + + return result +end + +""" + get_variable(run_info::Tuple, variable_name; kwargs...) + get_variable(run_info, variable_name; kwargs...) + +Get an array (or Tuple of arrays, if `run_info` is a Tuple) of the data for +`variable_name` from `run_info`. + +Some derived variables need to be calculated from the saved output, not just loaded from +file (with `postproc_load_variable`). This function takes care of that calculation, and +handles the case where `run_info` is a Tuple (which `postproc_load_data` does not handle). + +`kwargs...` are passed through to `postproc_load_variable()`. +""" +function get_variable end + +function get_variable(run_info::Tuple, variable_name; kwargs...) + return Tuple(get_variable(ri, variable_name; kwargs...) for ri ∈ run_info) +end + +function get_variable(run_info, variable_name; kwargs...) + if variable_name == "temperature" + vth = postproc_load_variable(run_info, "thermal_speed") + variable = vth.^2 + elseif variable_name == "collision_frequency" + n = postproc_load_variable(run_info, "density") + vth = postproc_load_variable(run_info, "thermal_speed") + variable = get_collision_frequency(run_info.collisions, n, vth) + elseif variable_name == "temperature_neutral" + vth = postproc_load_variable(run_info, "thermal_speed_neutral") + variable = vth.^2 + elseif variable_name == "sound_speed" + T_e = run_info.composition.T_e + T_i = get_variable(run_info, "temperature"; kwargs...) + + # Adiabatic index. Not too clear what value should be (see e.g. [Riemann 1991, + # below eq. (39)], or discussion of Bohm criterion in Stangeby's book. + gamma = 3.0 + + # Factor of 0.5 needed because temperatures are normalised to mi*cref^2, not Tref + variable = @. sqrt(0.5*(T_e + gamma*T_i)) + elseif variable_name == "mach_number" + upar = get_variable(run_info, "parallel_flow"; kwargs...) + cs = get_variable(run_info, "sound_speed"; kwargs...) + variable = upar ./ cs + else + variable = postproc_load_variable(run_info, variable_name) + end + + return variable +end + +""" +Read data which is a function of (z,r,t) or (z,r,species,t) + +run_names is a tuple. If it has more than one entry, this means that there are multiple +restarts (which are sequential in time), so concatenate the data from each entry together. +""" +function read_distributed_zr_data!(var::Array{mk_float,N}, var_name::String, + run_names::Tuple, file_key::String, nblocks::Tuple, + nz_local::mk_int,nr_local::mk_int,iskip::mk_int) where N + # dimension of var is [z,r,species,t] + + local_tind_start = 1 + local_tind_end = -1 + global_tind_start = 1 + global_tind_end = -1 + for (run_name, nb) in zip(run_names, nblocks) + for iblock in 0:nb-1 + fid = open_readonly_output_file(run_name,file_key,iblock=iblock,printout=false) + group = get_group(fid, "dynamic_data") + var_local = load_variable(group, var_name) + + ntime_local = size(var_local, N) + + # offset is the amount we have to skip at the beginning of this restart to + # line up properly with having outputs every iskip since the beginning of the + # first restart. + # Note: use rem(x,y,RoundDown) here because this gives a result that's + # definitely between 0 and y, whereas rem(x,y) or mod(x,y) give negative + # results for negative x. + offset = rem(1 - (local_tind_start-1), iskip, RoundDown) + if offset == 0 + # Actually want offset in the range [1,iskip], so correct if rem() + # returned 0 + offset = iskip + end + if local_tind_start > 1 + # The run being loaded is a restart (as local_tind_start=1 for the first + # run), so skip the first point, as this is a duplicate of the last point + # of the previous restart + offset += 1 + end + + local_tind_end = local_tind_start + ntime_local - 1 + global_tind_end = global_tind_start + length(offset:iskip:ntime_local) - 1 + + z_irank, z_nrank, r_irank, r_nrank = load_rank_data(fid) + + # min index set to avoid double assignment of repeated points + # 1 if irank = 0, 2 otherwise + imin_r = min(1,r_irank) + 1 + imin_z = min(1,z_irank) + 1 + for ir_local in imin_r:nr_local + for iz_local in imin_z:nz_local + ir_global = iglobal_func(ir_local,r_irank,nr_local) + iz_global = iglobal_func(iz_local,z_irank,nz_local) + if N == 4 + var[iz_global,ir_global,:,global_tind_start:global_tind_end] .= var_local[iz_local,ir_local,:,offset:iskip:end] + elseif N == 3 + var[iz_global,ir_global,global_tind_start:global_tind_end] .= var_local[iz_local,ir_local,offset:iskip:end] + else + error("Unsupported number of dimensions: $N") + end + end + end + close(fid) + end + local_tind_start = local_tind_end + 1 + global_tind_start = global_tind_end + 1 + end +end + +""" +""" +function construct_global_zr_coords(r_local, z_local) + + function make_global_input(coord_local) + return grid_input(coord_local.name, coord_local.ngrid, + coord_local.nelement_global, coord_local.nelement_global, 1, 0, coord_local.L, + coord_local.discretization, coord_local.fd_option, coord_local.cheb_option, coord_local.bc, + coord_local.advection, MPI.COMM_NULL, coord_local.element_spacing_option) + end + + r_global, r_global_spectral = define_coordinate(make_global_input(r_local)) + z_global, z_global_spectral = define_coordinate(make_global_input(z_local)) + + return r_global, r_global_spectral, z_global, z_global_spectral +end + end diff --git a/moment_kinetics/src/moment_kinetics.jl b/moment_kinetics/src/moment_kinetics.jl index 56c2cee32..70bdd4c3b 100644 --- a/moment_kinetics/src/moment_kinetics.jl +++ b/moment_kinetics/src/moment_kinetics.jl @@ -58,8 +58,8 @@ include("energy_equation.jl") include("force_balance.jl") include("source_terms.jl") include("numerical_dissipation.jl") -include("load_data.jl") include("moment_kinetics_input.jl") +include("load_data.jl") include("parameter_scans.jl") include("analysis.jl") include("utils.jl") diff --git a/moment_kinetics/test/restart_interpolation_tests.jl b/moment_kinetics/test/restart_interpolation_tests.jl index 251e6d4a9..f51ffcb5f 100644 --- a/moment_kinetics/test/restart_interpolation_tests.jl +++ b/moment_kinetics/test/restart_interpolation_tests.jl @@ -16,8 +16,8 @@ using moment_kinetics.load_data: open_readonly_output_file, load_coordinate_data load_neutral_particle_moments_data, load_neutral_pdf_data, load_time_data, load_species_data using moment_kinetics.interpolation: interpolate_to_grid_z, interpolate_to_grid_vpa -using moment_kinetics.makie_post_processing: get_run_info, close_run_info, - postproc_load_variable +using moment_kinetics.load_data: get_run_info_no_setup, close_run_info, + postproc_load_variable using moment_kinetics.type_definitions: mk_float include("nonlinear_sound_wave_inputs_and_expected_data.jl") @@ -136,7 +136,7 @@ function run_test(test_input, base, message, rtol, atol; tol_3V, kwargs...) # Read the output data path = joinpath(realpath(input["base_directory"]), name) - run_info = get_run_info((path, -1); dfns=true) + run_info = get_run_info_no_setup((path, -1); dfns=true) z = run_info.z z_spectral = run_info.z_spectral vpa = run_info.vpa diff --git a/plots_post_processing/src/plot_MMS_sequence.jl b/plots_post_processing/src/plot_MMS_sequence.jl index 21ba269b6..37554aa0d 100644 --- a/plots_post_processing/src/plot_MMS_sequence.jl +++ b/plots_post_processing/src/plot_MMS_sequence.jl @@ -16,7 +16,6 @@ using LaTeXStrings using ..post_processing_input: pp using ..plots_post_processing: compare_charged_pdf_symbolic_test, compare_fields_symbolic_test using ..plots_post_processing: compare_moments_symbolic_test, compare_neutral_pdf_symbolic_test -using ..plots_post_processing: read_distributed_zr_data!, construct_global_zr_coords using ..plots_post_processing: allocate_global_zr_neutral_moments, allocate_global_zr_charged_moments using ..plots_post_processing: allocate_global_zr_fields, get_geometry_and_composition using ..plots_post_processing: get_coords_nelement, get_coords_ngrid @@ -27,6 +26,7 @@ using moment_kinetics.load_data: load_fields_data, load_pdf_data using moment_kinetics.load_data: load_charged_particle_moments_data, load_neutral_particle_moments_data using moment_kinetics.load_data: load_neutral_pdf_data, load_time_data, load_species_data using moment_kinetics.load_data: load_block_data, load_coordinate_data, load_input +using moment_kinetics.load_data: read_distributed_zr_data!, construct_global_zr_coords using moment_kinetics.velocity_moments: integrate_over_vspace using moment_kinetics.manufactured_solns: manufactured_solutions, manufactured_electric_fields using moment_kinetics.moment_kinetics_input: mk_input, read_input_file diff --git a/plots_post_processing/src/plot_sequence.jl b/plots_post_processing/src/plot_sequence.jl index 8fbf8180b..c9e107026 100644 --- a/plots_post_processing/src/plot_sequence.jl +++ b/plots_post_processing/src/plot_sequence.jl @@ -11,7 +11,6 @@ using Statistics: mean using SpecialFunctions: erfi using LaTeXStrings # modules -using ..plots_post_processing: read_distributed_zr_data!, construct_global_zr_coords using ..plots_post_processing: allocate_global_zr_neutral_moments, allocate_global_zr_charged_moments using ..plots_post_processing: allocate_global_zr_fields#, get_coords_nelement using moment_kinetics.array_allocation: allocate_float @@ -20,6 +19,7 @@ using moment_kinetics.load_data: open_readonly_output_file using moment_kinetics.load_data: load_fields_data using moment_kinetics.load_data: load_time_data, load_species_data using moment_kinetics.load_data: load_block_data, load_coordinate_data +using moment_kinetics.load_data: read_distributed_zr_data!, construct_global_zr_coords using moment_kinetics.moment_kinetics_input: mk_input, read_input_file function plot_sequence_fields_data(path_list) diff --git a/plots_post_processing/src/plots_post_processing.jl b/plots_post_processing/src/plots_post_processing.jl index 9bf57ece8..8820cfb1f 100644 --- a/plots_post_processing/src/plots_post_processing.jl +++ b/plots_post_processing/src/plots_post_processing.jl @@ -40,7 +40,7 @@ using moment_kinetics.coordinates: define_coordinate using moment_kinetics.file_io: open_ascii_output_file using moment_kinetics.type_definitions: mk_float, mk_int using moment_kinetics.load_data: open_readonly_output_file, get_group, load_input, - load_time_data + load_time_data, construct_global_zr_coords using moment_kinetics.load_data: get_nranks using moment_kinetics.load_data: load_fields_data, load_pdf_data using moment_kinetics.load_data: load_charged_particle_moments_data, @@ -48,7 +48,7 @@ using moment_kinetics.load_data: load_charged_particle_moments_data, using moment_kinetics.load_data: load_distributed_charged_pdf_slice, load_distributed_neutral_pdf_slice, iglobal_func using moment_kinetics.load_data: load_neutral_pdf_data -using moment_kinetics.load_data: load_variable +using moment_kinetics.load_data: load_variable, read_distributed_zr_data! using moment_kinetics.load_data: load_coordinate_data, load_block_data, load_rank_data, load_species_data, load_mk_options using moment_kinetics.analysis: analyze_fields_data, analyze_moments_data, @@ -63,8 +63,7 @@ using moment_kinetics.input_structs: geometry_input, grid_input, species_composi using moment_kinetics.input_structs: electron_physics_type, boltzmann_electron_response, boltzmann_electron_response_with_simple_sheath using .post_processing_input: pp -using .shared_utils: calculate_and_write_frequencies, construct_global_zr_coords, - get_geometry_and_composition, read_distributed_zr_data! +using .shared_utils: calculate_and_write_frequencies, get_geometry_and_composition using TOML import Base: get From 15ab879a787823a3f8e616c7e15ecbe10cf0412c Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 19 Dec 2023 22:40:56 +0000 Subject: [PATCH 12/80] Comment out import of plot_test_data in fokker_planck_tests.jl This function is not available as Plots.jl is not included as a dependency of moment_kinetics any more. --- moment_kinetics/test/fokker_planck_tests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moment_kinetics/test/fokker_planck_tests.jl b/moment_kinetics/test/fokker_planck_tests.jl index feb930de2..d5c2b7ac6 100644 --- a/moment_kinetics/test/fokker_planck_tests.jl +++ b/moment_kinetics/test/fokker_planck_tests.jl @@ -16,7 +16,7 @@ using moment_kinetics.velocity_moments: get_density, get_upar, get_ppar, get_ppe using moment_kinetics.fokker_planck: init_fokker_planck_collisions_weak_form, fokker_planck_collision_operator_weak_form! using moment_kinetics.fokker_planck: conserving_corrections!, init_fokker_planck_collisions_direct_integration -using moment_kinetics.fokker_planck_test: print_test_data, plot_test_data, fkpl_error_data, allocate_error_data +using moment_kinetics.fokker_planck_test: print_test_data, fkpl_error_data, allocate_error_data #, plot_test_data using moment_kinetics.fokker_planck_test: F_Maxwellian, G_Maxwellian, H_Maxwellian using moment_kinetics.fokker_planck_test: d2Gdvpa2_Maxwellian, d2Gdvperp2_Maxwellian, d2Gdvperpdvpa_Maxwellian, dGdvperp_Maxwellian using moment_kinetics.fokker_planck_test: dHdvperp_Maxwellian, dHdvpa_Maxwellian, Cssp_Maxwellian_inputs From ea2236922a5ce0fa760416220b0a45c48be54439 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 19 Dec 2023 22:56:22 +0000 Subject: [PATCH 13/80] Update CI tests for reorganised package structure Also bumps the Julia version used for the CI to 1.9, as this is required in order to use package extensions. Also adds NCDatasets to test dependencies. This will require/install NCDatasets when running `Pkg.test()`, but it should still be possible to run the tests without NCDatasets by just doing `include("moment_kinetics/test/runtests.jl")`. --- .github/workflows/debug_checks.yml | 16 ++++++---------- .github/workflows/documentation.yml | 2 +- .github/workflows/examples.yml | 14 ++++---------- .github/workflows/longtest.yml | 20 +++++++------------- .github/workflows/parallel_test.yml | 21 +++++++-------------- .github/workflows/test.yml | 14 +++++--------- moment_kinetics/test/Project.toml | 1 + 7 files changed, 31 insertions(+), 57 deletions(-) diff --git a/.github/workflows/debug_checks.yml b/.github/workflows/debug_checks.yml index cbbc28ed5..a6113e7ec 100644 --- a/.github/workflows/debug_checks.yml +++ b/.github/workflows/debug_checks.yml @@ -18,27 +18,23 @@ jobs: - uses: mpi4py/setup-mpi@v1 with: mpi: 'openmpi' - - uses: actions/setup-python@v4 - with: - python-version: '3.11' - uses: julia-actions/setup-julia@v1 with: - version: '1.8' + version: '1.9' arch: x64 - uses: julia-actions/julia-buildpkg@v1 - env: - # Use the system Python for PyCall - avoids library linking error on macOS - PYTHON: "${{ env.pythonLocation }}/bin/python" + with: + project: 'moment_kinetics/' - name: Debug test run: | + cd moment_kinetics + # Hard code the debug level so that we can run without using the # `--compiled-modules=no` flag, which breaks Symbolics.jl at the # moment. sed -i -e "s/_debug_level = get_options.*/_debug_level = 2/" src/debugging.jl - pip3 install --user matplotlib - julia --project -e 'using MPIPreferences; MPIPreferences.use_system_binary()' - julia --project -e 'using Pkg; Pkg.build("MPI"; verbose=true)' + julia --project -e 'using MPIPreferences; MPIPreferences.use_system_binary(); using Pkg; Pkg.precompile()' # Need to use openmpi so that the following arguments work: # * `--mca rmaps_base_oversubscribe 1` allows oversubscription (more processes diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index ea1f941b1..c1117cf02 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -18,7 +18,7 @@ jobs: - name: Install dependencies run: | pip3 install --user matplotlib - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.develop(PackageSpec(path=joinpath(pwd(), "makie_post_processing"))); Pkg.develop(PackageSpec(path=joinpath(pwd(), "plots_post_processing"))); Pkg.instantiate()' + julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=joinpath(pwd(), moment_kinetics))); Pkg.develop(PackageSpec(path=joinpath(pwd(), "makie_post_processing"))); Pkg.develop(PackageSpec(path=joinpath(pwd(), "plots_post_processing"))); Pkg.instantiate()' - name: Build and deploy env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Authenticate with GitHub Actions token diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 36270210a..e3afd3099 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -14,19 +14,13 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: '3.11' - uses: julia-actions/setup-julia@v1 with: - version: '1.8' + version: '1.9' arch: x64 - - uses: julia-actions/julia-buildpkg@v1 - env: - # Use the system Python for PyCall - avoids library linking error on macOS - PYTHON: "${{ env.pythonLocation }}/bin/python" - name: Test examples run: | - pip3 install --user matplotlib + touch Project.toml + julia -O3 --project -e 'import Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.add("NCDatasets"); Pkg.precompile()' # Reduce nstep for each example to 10 to avoid the CI job taking too long - julia --project -O3 -e 'using moment_kinetics; for (root, dirs, files) in walkdir("examples") for file in files if endswith(file, ".toml") filename = joinpath(root, file); println(filename); input = moment_kinetics.moment_kinetics_input.read_input_file(filename); input["nstep"] = 10; run_moment_kinetics(input) end end end' + julia -O3 --project -e 'using moment_kinetics; for (root, dirs, files) in walkdir("examples") for file in files if endswith(file, ".toml") filename = joinpath(root, file); println(filename); input = moment_kinetics.moment_kinetics_input.read_input_file(filename); input["nstep"] = 10; run_moment_kinetics(input) end end end' diff --git a/.github/workflows/longtest.yml b/.github/workflows/longtest.yml index eb001ce42..8135e356c 100644 --- a/.github/workflows/longtest.yml +++ b/.github/workflows/longtest.yml @@ -19,22 +19,16 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: '3.11' - uses: julia-actions/setup-julia@v1 with: - version: '1.8' + version: '1.9' arch: x64 - uses: julia-actions/julia-buildpkg@v1 - env: - # Use the system Python for PyCall - avoids library linking error on macOS - PYTHON: "${{ env.pythonLocation }}/bin/python" - # The following is copied and simplified from - # https://github.com/julia-actions/julia-runtest/blob/master/action.yml - # in order to pass customised arguments to `Pkg.test()` - # + with: + project: 'moment_kinetics/' + # The following is copied and simplified from + # https://github.com/julia-actions/julia-runtest/blob/master/action.yml + # in order to pass customised arguments to `Pkg.test()` - run: | - pip3 install --user matplotlib - julia --check-bounds=yes --color=yes --depwarn=yes --project=@. -e 'import Pkg; Pkg.test(; test_args=["--long"])' + julia --check-bounds=yes --color=yes --depwarn=yes --project=moment_kinetics/ -e 'import Pkg; Pkg.test(; test_args=["--long"])' shell: bash diff --git a/.github/workflows/parallel_test.yml b/.github/workflows/parallel_test.yml index a2d4a3995..8e2507ed1 100644 --- a/.github/workflows/parallel_test.yml +++ b/.github/workflows/parallel_test.yml @@ -10,7 +10,7 @@ jobs: matrix: os: [ubuntu-latest, macOS-latest] include: - - julia_version: '1.8' + - julia_version: '1.9' fail-fast: false timeout-minutes: 120 @@ -19,29 +19,22 @@ jobs: - uses: mpi4py/setup-mpi@v1 with: mpi: 'openmpi' - - uses: actions/setup-python@v4 - with: - python-version: '3.11' - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.julia_version }} arch: x64 - - uses: julia-actions/julia-buildpkg@v1 - env: - # Use the system Python for PyCall - avoids library linking error on macOS - PYTHON: "${{ env.pythonLocation }}/bin/python" - run: | - pip3 install --user matplotlib - julia --project -e 'using MPIPreferences; MPIPreferences.use_system_binary()' - julia --project -e 'using Pkg; Pkg.build("MPI"; verbose=true)' + touch Project.toml + julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.add(["MPI", "MPIPreferences", "NCDatasets", "Random", "SpecialFunctions", "Test"]); Pkg.develop(path="moment_kinetics/"); using MPIPreferences; MPIPreferences.use_system_binary()' + julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.precompile()' # Need to use openmpi so that the following arguments work: # * `--mca rmaps_base_oversubscribe 1` allows oversubscription (more processes # than physical cores). # * `--mca mpi_yield_when_idle 1` changes a setting to prevent excessively # terrible performance when oversubscribing. - mpiexec -np 3 --mca rmaps_base_oversubscribe 1 julia --project=@. -O3 --check-bounds=no test/runtests.jl --debug 1 - mpiexec -np 4 --mca rmaps_base_oversubscribe 1 julia --project=@. -O3 --check-bounds=no test/runtests.jl --debug 1 - mpiexec -np 2 --mca rmaps_base_oversubscribe 1 julia --project=@. -O3 --check-bounds=no test/runtests.jl --debug 1 --long + mpiexec -np 3 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 + mpiexec -np 4 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 + mpiexec -np 2 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 --long # Note: MPI.jl's default implementation is mpich, which has a similar option # `--with-device=ch3:sock`, but that needs to be set when compiling mpich. shell: bash diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 363220a10..736004e82 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,17 +14,13 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: '3.11' - uses: julia-actions/setup-julia@v1 with: - version: '1.8' + version: '1.9' arch: x64 - uses: julia-actions/julia-buildpkg@v1 - env: - # Use the system Python for PyCall - avoids library linking error on macOS - PYTHON: "${{ env.pythonLocation }}/bin/python" - - run: | - pip3 install --user matplotlib + with: + project: 'moment_kinetics/' - uses: julia-actions/julia-runtest@v1 + with: + project: 'moment_kinetics/' diff --git a/moment_kinetics/test/Project.toml b/moment_kinetics/test/Project.toml index fb1cb33d4..f273d69c2 100644 --- a/moment_kinetics/test/Project.toml +++ b/moment_kinetics/test/Project.toml @@ -1,6 +1,7 @@ [deps] ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" +NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" PyPlot = "d330b81b-6aea-500a-939a-2ce795aea3ee" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" From 526e4aeef983e0e95f114e341dd84b55c99dc840 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 20 Dec 2023 09:46:41 +0000 Subject: [PATCH 14/80] Modernise test dependency specification, remove unneeded dependencies The 'modern' way to specify extra dependencies for the tests in Julia is using `test` in the `[targets]` section of the top-level Project.toml. The old method of using an extra `test/Project.toml` is described as 'the old method' which, while it will continue to be supported in Julia-1.x, seems to be out of favour. The main point of this update though is to remove extra dependencies from the tests, which are not needed. --- moment_kinetics/Project.toml | 7 +++++++ moment_kinetics/test/Project.toml | 10 ---------- 2 files changed, 7 insertions(+), 10 deletions(-) delete mode 100644 moment_kinetics/test/Project.toml diff --git a/moment_kinetics/Project.toml b/moment_kinetics/Project.toml index ed20aba82..fde1b1797 100644 --- a/moment_kinetics/Project.toml +++ b/moment_kinetics/Project.toml @@ -49,3 +49,10 @@ manufactured_solns_ext = "Symbolics" [compat] HDF5_jll = "<1.14, >=1.15" julia = "1.7.0" + +[extras] +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Random", "Test", "NCDatasets"] diff --git a/moment_kinetics/test/Project.toml b/moment_kinetics/test/Project.toml deleted file mode 100644 index f273d69c2..000000000 --- a/moment_kinetics/test/Project.toml +++ /dev/null @@ -1,10 +0,0 @@ -[deps] -ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" -MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" -NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" -PyPlot = "d330b81b-6aea-500a-939a-2ce795aea3ee" -Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" -LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" From 7e71e5042fa17b22c2c6faefd00e960255a1df75 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 20 Dec 2023 13:45:19 +0000 Subject: [PATCH 15/80] Allow tests to run when optional dependencies are not installed NCDatasets is now an optional dependency (also Symbolics, but that is not used in the test suite). To allow the tests to run when NCDatasets is not installed, the tests that use NetCDF now fall back to HDF5 if NetCDF is not available. The fall-back can be disabled explicitly by passing the --force-optional-dependencies command line flag, to ensure that the CI tests do actually use NetCDF I/O, and will error if it is not available. --- .github/workflows/longtest.yml | 2 +- .github/workflows/parallel_test.yml | 6 +++--- .github/workflows/test.yml | 9 ++++++--- moment_kinetics/src/command_line_options.jl | 3 +++ moment_kinetics/test/setup.jl | 5 +++-- moment_kinetics/test/sound_wave_tests.jl | 9 ++++++++- 6 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/longtest.yml b/.github/workflows/longtest.yml index 8135e356c..9c679c2c8 100644 --- a/.github/workflows/longtest.yml +++ b/.github/workflows/longtest.yml @@ -30,5 +30,5 @@ jobs: # https://github.com/julia-actions/julia-runtest/blob/master/action.yml # in order to pass customised arguments to `Pkg.test()` - run: | - julia --check-bounds=yes --color=yes --depwarn=yes --project=moment_kinetics/ -e 'import Pkg; Pkg.test(; test_args=["--long"])' + julia --check-bounds=yes --color=yes --depwarn=yes --project=moment_kinetics/ -e 'import Pkg; Pkg.test(; test_args=["--long", "--force-optional-dependencies"])' shell: bash diff --git a/.github/workflows/parallel_test.yml b/.github/workflows/parallel_test.yml index 8e2507ed1..601726dc1 100644 --- a/.github/workflows/parallel_test.yml +++ b/.github/workflows/parallel_test.yml @@ -32,9 +32,9 @@ jobs: # than physical cores). # * `--mca mpi_yield_when_idle 1` changes a setting to prevent excessively # terrible performance when oversubscribing. - mpiexec -np 3 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 - mpiexec -np 4 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 - mpiexec -np 2 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 --long + mpiexec -np 3 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 --force-optional-dependencies + mpiexec -np 4 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 --force-optional-dependencies + mpiexec -np 2 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 --long --force-optional-dependencies # Note: MPI.jl's default implementation is mpich, which has a similar option # `--with-device=ch3:sock`, but that needs to be set when compiling mpich. shell: bash diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 736004e82..ff5a21ae1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,9 @@ jobs: - uses: julia-actions/julia-buildpkg@v1 with: project: 'moment_kinetics/' - - uses: julia-actions/julia-runtest@v1 - with: - project: 'moment_kinetics/' + # The following is copied and simplified from + # https://github.com/julia-actions/julia-runtest/blob/master/action.yml + # in order to pass customised arguments to `Pkg.test()` + - run: | + julia --check-bounds=yes --color=yes --depwarn=yes --project=moment_kinetics/ -e 'import Pkg; Pkg.test(; test_args=["--force-optional-dependencies"])' + shell: bash diff --git a/moment_kinetics/src/command_line_options.jl b/moment_kinetics/src/command_line_options.jl index 430fbb8da..3586927c4 100644 --- a/moment_kinetics/src/command_line_options.jl +++ b/moment_kinetics/src/command_line_options.jl @@ -33,6 +33,9 @@ const s = ArgParseSettings() arg_type = Int default = -1 # Options for tests + "--force-optional-dependencies" + help = "Skip workarounds that allow tests to run without optional dependencies" + action = :store_true "--long" help = "Include more tests, increasing test run time." action = :store_true diff --git a/moment_kinetics/test/setup.jl b/moment_kinetics/test/setup.jl index 05d4c1171..3bb9896ef 100644 --- a/moment_kinetics/test/setup.jl +++ b/moment_kinetics/test/setup.jl @@ -12,8 +12,8 @@ using moment_kinetics module MKTestUtilities -export use_verbose, @long, quietoutput, get_MPI_tempdir, global_rank, maxabs_norm, - @testset_skip +export use_verbose, force_optional_dependencies, @long, quietoutput, get_MPI_tempdir, + global_rank, maxabs_norm, @testset_skip using moment_kinetics.communication: comm_world, global_rank using moment_kinetics.command_line_options: get_options @@ -21,6 +21,7 @@ using moment_kinetics.command_line_options: get_options using MPI const use_verbose = get_options()["verbose"] +const force_optional_dependencies = get_options()["force-optional-dependencies"] # Convenience modifiers for test calls diff --git a/moment_kinetics/test/sound_wave_tests.jl b/moment_kinetics/test/sound_wave_tests.jl index 4d6ea933e..9ffdf25ed 100644 --- a/moment_kinetics/test/sound_wave_tests.jl +++ b/moment_kinetics/test/sound_wave_tests.jl @@ -6,6 +6,8 @@ using Base.Filesystem: tempname #using Plots: plot, plot!, gui using moment_kinetics.array_allocation: allocate_float +using moment_kinetics.input_structs: netcdf +using moment_kinetics.file_io: io_has_implementation using moment_kinetics.load_data: open_readonly_output_file using moment_kinetics.load_data: load_fields_data, load_time_data using moment_kinetics.load_data: load_species_data, load_coordinate_data @@ -16,6 +18,11 @@ const analytical_rtol = 3.e-2 const regression_rtol = 1.e-14 const regression_range = 5:10 +# Use "netcdf" to test the NetCDF I/O if it is available (or if we are forcing optional +# dependencies to be used, e.g. for CI tests), otherwise fall back to "hdf5". +const binary_format = (force_optional_dependencies || io_has_implementation(netcdf)) ? + "netcdf" : "hdf5" + # default inputs for tests test_input_finite_difference = Dict("n_ion_species" => 1, "n_neutral_species" => 1, @@ -74,7 +81,7 @@ test_input_finite_difference = Dict("n_ion_species" => 1, "vz_L" => 8.0, "vz_bc" => "periodic", "vz_discretization" => "finite_difference", - "output" => Dict{String,Any}("binary_format" => "netcdf") + "output" => Dict{String,Any}("binary_format" => binary_format) ) test_input_finite_difference_split_1_moment = From 521bb723ad7e4efeabff52e192b849d461792eda Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 20 Dec 2023 14:51:20 +0000 Subject: [PATCH 16/80] Update machine setup scripts with new HDF5.jl setup function --- machines/archer/machine_setup_stage_two.jl | 5 +++-- machines/marconi/machine_setup_stage_two.jl | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/machines/archer/machine_setup_stage_two.jl b/machines/archer/machine_setup_stage_two.jl index 88e4cdac0..ea5049b57 100644 --- a/machines/archer/machine_setup_stage_two.jl +++ b/machines/archer/machine_setup_stage_two.jl @@ -12,8 +12,9 @@ Pkg.resolve() ############ println("\n** Setting up to use system HDF5\n") -ENV["JULIA_HDF5_PATH"] = ENV["HDF5_DIR"] # system hdf5 -Pkg.build() +hdf5_dir = ENV["HDF5_DIR"] # system hdf5 +using HDF5 +HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), joinpath(hdf5_dir, "libhdf5_hl.so")) # MPI setup diff --git a/machines/marconi/machine_setup_stage_two.jl b/machines/marconi/machine_setup_stage_two.jl index 4e64fb8f3..e9b550a21 100644 --- a/machines/marconi/machine_setup_stage_two.jl +++ b/machines/marconi/machine_setup_stage_two.jl @@ -15,8 +15,9 @@ artifact_dir = joinpath(repo_dir, "machines", "artifacts") ############ println("\n** Setting up to use custom compiled HDF5\n") -ENV["JULIA_HDF5_PATH"] = joinpath(artifact_dir, "hdf5-build/") -Pkg.build() +hdf5_dir = joinpath(artifact_dir, "hdf5-build/") +using HDF5 +HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), joinpath(hdf5_dir, "libhdf5_hl.so")) # MPI setup From 82084cc7b894c5b9573c161718b3489e38fffbd5 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 20 Dec 2023 17:24:56 +0000 Subject: [PATCH 17/80] Reduce number of tests in parallel on macOS macOS parallel tests are very slow (at the moment), so only run on 4 processes (not 3 or 2), and do not run long tests. --- .github/workflows/parallel_test.yml | 40 ++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/parallel_test.yml b/.github/workflows/parallel_test.yml index 601726dc1..b2734b1db 100644 --- a/.github/workflows/parallel_test.yml +++ b/.github/workflows/parallel_test.yml @@ -4,14 +4,8 @@ name: Run tests in parallel on: [push, pull_request, workflow_dispatch] jobs: - test: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macOS-latest] - include: - - julia_version: '1.9' - fail-fast: false + test-ubuntu: + runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -21,7 +15,7 @@ jobs: mpi: 'openmpi' - uses: julia-actions/setup-julia@v1 with: - version: ${{ matrix.julia_version }} + version: '1.9' arch: x64 - run: | touch Project.toml @@ -38,3 +32,31 @@ jobs: # Note: MPI.jl's default implementation is mpich, which has a similar option # `--with-device=ch3:sock`, but that needs to be set when compiling mpich. shell: bash + + # macOS is slow at the moment, so only run one set of parallel tests + test-macOS: + runs-on: macOS-latest + timeout-minutes: 120 + + steps: + - uses: actions/checkout@v4 + - uses: mpi4py/setup-mpi@v1 + with: + mpi: 'openmpi' + - uses: julia-actions/setup-julia@v1 + with: + version: '1.9' + arch: x64 + - run: | + touch Project.toml + julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.add(["MPI", "MPIPreferences", "NCDatasets", "Random", "SpecialFunctions", "Test"]); Pkg.develop(path="moment_kinetics/"); using MPIPreferences; MPIPreferences.use_system_binary()' + julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.precompile()' + # Need to use openmpi so that the following arguments work: + # * `--mca rmaps_base_oversubscribe 1` allows oversubscription (more processes + # than physical cores). + # * `--mca mpi_yield_when_idle 1` changes a setting to prevent excessively + # terrible performance when oversubscribing. + mpiexec -np 4 --mca rmaps_base_oversubscribe 1 julia --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --debug 1 --force-optional-dependencies + # Note: MPI.jl's default implementation is mpich, which has a similar option + # `--with-device=ch3:sock`, but that needs to be set when compiling mpich. + shell: bash From 52067412feb2a50752d5d9add49f01bf1363cccc Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 21 Dec 2023 10:30:41 +0000 Subject: [PATCH 18/80] Update HDF5 installed for Marconi setup to 1.14.3 --- machines/marconi/compile_dependencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machines/marconi/compile_dependencies.sh b/machines/marconi/compile_dependencies.sh index 454a65f78..3bd760726 100755 --- a/machines/marconi/compile_dependencies.sh +++ b/machines/marconi/compile_dependencies.sh @@ -25,9 +25,9 @@ if [ -d hdf5-build ]; then fi if [ $BUILDHDF5 -eq 0 ]; then - HDF5=hdf5-1.14.1-2 + HDF5=hdf5-1.14.3 # Download and extract the source - wget -O ${HDF5}.tar.bz2 https://www.hdfgroup.org/package/hdf5-1-14-1-2-tar-bz2/?wpdmdl=17997 + wget -O ${HDF5}.tar.bz2 https://www.hdfgroup.org/package/hdf5-1-14-3-tar-bz2/?wpdmdl=18469 tar xjf ${HDF5}.tar.bz2 cd $HDF5 From 519879aa293d20c2ce28769a77774c00edcccba9 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 20 Dec 2023 17:41:41 +0000 Subject: [PATCH 19/80] Update machine setup scripts for restructured packages, add generic-pc Updates the machine setup scripts in the `machines/` subdirectory and the precompilation scripts to account for the new package structure (with postprocessing tools moved into separate packages). Also adds a 'generic-pc' option that can be used to set up moment_kinetics on a standard Linux desktop/laptop machine. --- .../jobscript-postprocess-plotsjl.template | 2 +- .../archer/jobscript-postprocess.template | 2 +- ...precompile-makie-post-processing.template} | 2 +- ...-precompile-plots-post-processing.template | 27 ++ machines/generic-pc/compile_dependencies.sh | 57 ++++ .../generic-pc/machine_setup_stage_two.jl | 114 +++++++ machines/machine_setup.sh | 306 ++++++++++++------ machines/marconi/compile_dependencies.sh | 2 +- .../jobscript-postprocess-plotsjl.template | 2 +- .../marconi/jobscript-postprocess.template | 2 +- ...precompile-makie-post-processing.template} | 2 +- ...-precompile-plots-post-processing.template | 20 ++ machines/shared/machine_setup.jl | 95 ++++-- precompile-makie-post-processing-submit.sh | 37 +++ precompile-makie-post-processing.jl | 3 - precompile-plots-post-processing-submit.sh | 37 +++ precompile-plots-post-processing.jl | 13 + precompile-submit.sh | 8 - run_makie_post_processing.jl | 10 +- run_post_processing.jl | 10 +- util/precompile_makie_plots.jl | 6 +- 21 files changed, 602 insertions(+), 155 deletions(-) rename machines/archer/{jobscript-precompile-postprocessing.template => jobscript-precompile-makie-post-processing.template} (86%) create mode 100644 machines/archer/jobscript-precompile-plots-post-processing.template create mode 100755 machines/generic-pc/compile_dependencies.sh create mode 100644 machines/generic-pc/machine_setup_stage_two.jl rename machines/marconi/{jobscript-precompile-postprocessing.template => jobscript-precompile-makie-post-processing.template} (77%) create mode 100644 machines/marconi/jobscript-precompile-plots-post-processing.template create mode 100755 precompile-makie-post-processing-submit.sh create mode 100755 precompile-plots-post-processing-submit.sh create mode 100644 precompile-plots-post-processing.jl diff --git a/machines/archer/jobscript-postprocess-plotsjl.template b/machines/archer/jobscript-postprocess-plotsjl.template index a46599cba..4f43a2f77 100644 --- a/machines/archer/jobscript-postprocess-plotsjl.template +++ b/machines/archer/jobscript-postprocess-plotsjl.template @@ -19,6 +19,6 @@ source julia.env export SRUN_CPUS_PER_TASK=$SLURM_CPUS_PER_TASK echo "post-processing (with original post_processing) RUNDIR $(date)" -bin/julia -Jmoment_kinetics.so --project run_post_processing.jl RUNDIR +bin/julia -Jplots_postproc.so --project=plots_post_processing/ run_post_processing.jl RUNDIR echo "finished post-processing RUNDIR $(date)" diff --git a/machines/archer/jobscript-postprocess.template b/machines/archer/jobscript-postprocess.template index 37bd716ac..f63441bff 100644 --- a/machines/archer/jobscript-postprocess.template +++ b/machines/archer/jobscript-postprocess.template @@ -19,6 +19,6 @@ source julia.env export SRUN_CPUS_PER_TASK=$SLURM_CPUS_PER_TASK echo "post-processing RUNDIR $(date)" -bin/julia -Jmakie_postproc.so --project run_makie_post_processing.jl RUNDIR +bin/julia -Jmakie_postproc.so --project=makie_post_processing/ run_makie_post_processing.jl RUNDIR echo "finished post-processing RUNDIR $(date)" diff --git a/machines/archer/jobscript-precompile-postprocessing.template b/machines/archer/jobscript-precompile-makie-post-processing.template similarity index 86% rename from machines/archer/jobscript-precompile-postprocessing.template rename to machines/archer/jobscript-precompile-makie-post-processing.template index bb4950ce9..44f70f5f1 100644 --- a/machines/archer/jobscript-precompile-postprocessing.template +++ b/machines/archer/jobscript-precompile-makie-post-processing.template @@ -22,6 +22,6 @@ export SRUN_CPUS_PER_TASK=$SLURM_CPUS_PER_TASK echo "precompiling post-processing $(date)" -bin/julia --project precompile-makie-post-processing.jl +bin/julia --project=makie_post_processing/ precompile-makie-post-processing.jl echo "finished!" diff --git a/machines/archer/jobscript-precompile-plots-post-processing.template b/machines/archer/jobscript-precompile-plots-post-processing.template new file mode 100644 index 000000000..6d7ae4995 --- /dev/null +++ b/machines/archer/jobscript-precompile-plots-post-processing.template @@ -0,0 +1,27 @@ +#!/bin/bash + +#SBATCH --mem=64G +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 +#SBATCH --time=1:00:00 +#SBATCH --account=ACCOUNT +#SBATCH --partition=serial +#SBATCH --qos=serial +#SBATCH --output=PRECOMPILEDIRslurm-%j.out + +set -e + +cd $SLURM_SUBMIT_DIR + +# Get setup for Julia +source julia.env + +# Workaround as cpus-per-task no longer inherited by srun from sbatch. +# See https://docs.archer2.ac.uk/faq/upgrade-2023/ +export SRUN_CPUS_PER_TASK=$SLURM_CPUS_PER_TASK + +echo "precompiling post-processing $(date)" + +bin/julia --project=plots_post_processing/ precompile-plots-post-processing.jl + +echo "finished!" diff --git a/machines/generic-pc/compile_dependencies.sh b/machines/generic-pc/compile_dependencies.sh new file mode 100755 index 000000000..2c2d107c6 --- /dev/null +++ b/machines/generic-pc/compile_dependencies.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -e + +cd machines/artifacts/ +ARTIFACT_DIR=$PWD + +# HDF5 +###### + +echo "Do you want to download, and compile a local version of HDF5 (if you do" +echo "not do this, you will be given the option to choose an HDF5 library to" +echo "link later)? [y]/n" +read -p "> " input +while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input +done +if [[ -z $input || $input == "y" ]]; then + BUILDHDF5=0 +else + BUILDHDF5=1 +fi + +if [[ BUILDHDF5 -eq 1 && -d hdf5-build ]]; then + rm -r hdf5-build +elif [[ BUILDHDF5 -eq 0 && -d hdf5-build ]]; then + echo "HDF5 appears to have been downloaded, compiled and installed already." + echo "Do you want to download, compile and install again, overwriting the existing " + echo "version? y/[n]" + read -p "> " input + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + done + if [[ -z $input || $input == "n" ]]; then + BUILDHDF5=1 + fi +fi + +if [ $BUILDHDF5 -eq 0 ]; then + HDF5=hdf5-1.14.3 + # Download and extract the source + wget -O ${HDF5}.tar.bz2 https://www.hdfgroup.org/package/hdf5-1-14-3-tar-bz2/?wpdmdl=18469 + tar xjf ${HDF5}.tar.bz2 + + cd $HDF5 + + # Configure and compile + CC=mpicc ./configure --enable-parallel --prefix=$ARTIFACT_DIR/hdf5-build/ + make -j 4 + make install +fi + +exit 0 diff --git a/machines/generic-pc/machine_setup_stage_two.jl b/machines/generic-pc/machine_setup_stage_two.jl new file mode 100644 index 000000000..936d098b9 --- /dev/null +++ b/machines/generic-pc/machine_setup_stage_two.jl @@ -0,0 +1,114 @@ +using Pkg + +""" + get_input_with_path_completion(message=nothing) + +Print `message` and get user input using the `read` utility from bash to allow +path-completion. + +Solution adapted from +https://discourse.julialang.org/t/collecting-all-output-from-shell-commands/15592/2 +""" +function get_input_with_path_completion(message=nothing) + if message !== nothing + println(message) + end + + # The `out` object is used by `pipeline()` to capture output from the shell command. + out = Pipe() + + # Use Julia's shell command functionality to actually run bash - not sure how to use + # `read` and get output from it without using bash. + run(pipeline(`bash -c "read -e -p '> ' USERINPUT; echo \$USERINPUT"`, stdout=out)) + + # Need to close `out.in` to be able to read from `out`. + close(out.in) + + # chomp removes the trailing '\n'. 'String()' converts Vector{Char} to a String. + input = chomp(String(read(out))) + + return input +end + +# Instantiate packages so we can use MPIPreferences below +######################################################### + +println("\n** Getting dependencies\n") +Pkg.instantiate() +Pkg.resolve() + + +# HDF5 setup +############ + +local_hdf5_install_dir = joinpath("machines", "artifacts", "hdf5-build", "lib") +if isdir(local_hdf5_install_dir) + local_hdf5_install_dir = realpath(local_hdf5_install_dir) + # We have downloaded and compiled HDF5, so link that + hdf5_dir = local_hdf5_install_dir + hdf5_lib = joinpath(local_hdf5_install_dir, "libhdf5.so") + hdf5_lib_hl = joinpath(local_hdf5_install_dir, "libhdf5_hl.so") +else + println("\n** Setting up to use system HDF5\n") + default_hdf5_dir = get(ENV, "HDF5_DIR", "") # try to find a path to a system hdf5, may not work on all systems + hdf5_dir = "" + hdf5_lib = "" + hdf5_lib_hl = "" + while true + global hdf5_dir, hdf5_lib, hdf5_lib_hl + hdf5_dir = get_input_with_path_completion( + "\nAn HDF5 installation compiled with your system MPI is required to use\n" + * "parallel I/O. Enter the directory where the libhdf5.so and libhdf5_hl.so are\n" + * "located (enter 'default' to use the Julia-provided HDF5, which does not\n" + * "support parallel I/O): [$default_hdf5_dir]") + + if hdf5_dir == "" + hdf5_dir = default_hdf5_dir + end + + if hdf5_dir == "default" + break + end + + if isdir(hdf5_dir) + hdf5_dir = realpath(hdf5_dir) + end + hdf5_lib = joinpath(hdf5_dir, "libhdf5.so") + hdf5_lib_hl = joinpath(hdf5_dir, "libhdf5_hl.so") + if isfile(hdf5_lib) && isfile(hdf5_lib_hl) + break + else + # Remove trailing slash if it exists so that we can print a single trailing slash + # consistently + hdf5_dir = rstrip(hdf5_dir, '/') + print("HDF5 libraries not found in '$hdf5_dir/'.") + if !isfile(hdf5_lib) + print(" $hdf5_lib does not exist.") + end + if !isfile(hdf5_lib_hl) + print(" $hdf5_lib_hl does not exist.") + end + end + end +end +if hdf5_dir != "default" + using HDF5 + HDF5.API.set_libraries!(hdf5_lib, hdf5_lib_hl) +end + +# MPI setup +########### + +println("\n** Setting up to use system MPI\n") +using MPIPreferences +MPIPreferences.use_system_binary() + + +# Force exit so Julia must be restarted +####################################### + +println() +println("************************************************************") +println("Julia must be restarted to use the updated MPI, exiting now.") +println("************************************************************") +exit(0) diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 7114848d9..4d9c9102c 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -24,6 +24,10 @@ while getopts "hd" opt; do esac done +# Ensure a Project.toml exists in the top-level directory, which defines a +# project environment that we can install packages to. +touch Project.toml + # Get the positional argument as JULIA # [See https://stackoverflow.com/a/13400237] JULIA=${@:$OPTIND:1} @@ -36,7 +40,7 @@ if ls /marconi > /dev/null 2>&1; then elif module avail 2>&1 | grep -q epcc; then DEFAULT_MACHINE=archer else - DEFAULT_MACHINE= + DEFAULT_MACHINE=generic-pc fi # Create directory to set up Python venv and download/compile dependencies in @@ -178,6 +182,46 @@ done echo "Setting up for '$MACHINE'" echo +if [[ $MACHINE -eq "generic-pc" ]]; then + BATCH_SYSTEM=1 +else + BATCH_SYSTEM=0 +fi + +echo "Would you like to set up makie_post_processing? y/[n]" +read -p "> " input +echo +while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + # $input must be empty, 'y' or 'n'. It is none of these, so ask for input + # again until we get a valid response. + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + echo +done +if [[ $input == "y" ]]; then + USE_MAKIE_POSTPROC=0 +else + USE_MAKIE_POSTPROC=1 +fi + +echo "Would you like to set up plots_post_processing? y/[n]" +read -p "> " input +echo +while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + # $input must be empty, 'y' or 'n'. It is none of these, so ask for input + # again until we get a valid response. + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + echo +done +if [[ $input == "y" ]]; then + USE_PLOTS_POSTPROC=0 +else + USE_PLOTS_POSTPROC=1 +fi + # Note that the $(eval echo )) is needed to remove quotes around # arguments. Adding the quotes in the Julia script is necessary so that if an # argument is empty it is not lost when parsing the Julia script output into @@ -189,12 +233,16 @@ DEFAULT_POSTPROC_MEMORY=$(eval echo ${DEFAULTS[3]}) DEFAULT_PARTITION=$(eval echo ${DEFAULTS[4]}) DEFAULT_QOS=$(eval echo ${DEFAULTS[5]}) -# Get the account to submit jobs with -echo "Enter the account code used to submit jobs []:" -read -p "> " ACCOUNT -echo -echo "Account code used is $ACCOUNT" -echo +if [[ $BATCH_SYSTEM -eq 0 ]]; then + # Get the account to submit jobs with + echo "Enter the account code used to submit jobs []:" + read -p "> " ACCOUNT + echo + echo "Account code used is $ACCOUNT" + echo +else + ACCOUNT="" +fi # Get the location for the .julia directory, in case this has to have a # non-default value, e.g. because the user's home directory is not accessible @@ -225,128 +273,192 @@ echo echo "Using julia_directory=$JULIA_DIRECTORY" echo -# Get the setting for the default run time -echo "Enter the default value for the time limit for simulation jobs [$DEFAULT_RUN_TIME]:" -read -p "> " input -if [ ! -z "$input" ]; then - DEFAULT_RUN_TIME=$input -fi -echo -echo "Default simulation time limit is $DEFAULT_RUN_TIME" -echo +if [[ $BATCH_SYSTEM -eq 0 ]]; then + # Get the setting for the default run time + echo "Enter the default value for the time limit for simulation jobs [$DEFAULT_RUN_TIME]:" + read -p "> " input + if [ ! -z "$input" ]; then + DEFAULT_RUN_TIME=$input + fi + echo + echo "Default simulation time limit is $DEFAULT_RUN_TIME" + echo -# Get the setting for the default number of nodes -echo "Enter the default value for the number of nodes for a run [$DEFAULT_NODES]:" -read -p "> " input -if [ ! -z "$input" ]; then - DEFAULT_NODES=$input -fi -echo -echo "Default number of nodes is $DEFAULT_NODES" -echo + # Get the setting for the default number of nodes + echo "Enter the default value for the number of nodes for a run [$DEFAULT_NODES]:" + read -p "> " input + if [ ! -z "$input" ]; then + DEFAULT_NODES=$input + fi + echo + echo "Default number of nodes is $DEFAULT_NODES" + echo -# Get the setting for the default postproc time -echo "Enter the default value for the time limit for post-processing jobs [$DEFAULT_POSTPROC_TIME]:" -read -p "> " input -if [ ! -z "$input" ]; then - DEFAULT_POSTPROC_TIME=$input -fi -echo -echo "Default post-processing time limit is $DEFAULT_POSTPROC_TIME" -echo + # Get the setting for the default postproc time + echo "Enter the default value for the time limit for post-processing jobs [$DEFAULT_POSTPROC_TIME]:" + read -p "> " input + if [ ! -z "$input" ]; then + DEFAULT_POSTPROC_TIME=$input + fi + echo + echo "Default post-processing time limit is $DEFAULT_POSTPROC_TIME" + echo -# Get the setting for the default postproc memory -echo "Enter the default value for the memory requested for post-processing jobs [$DEFAULT_POSTPROC_MEMORY]:" -read -p "> " input -if [ ! -z "$input" ]; then - DEFAULT_POSTPROC_MEMORY=$input -fi -echo -echo "Default post-processing memory requested is $DEFAULT_POSTPROC_MEMORY" -echo + # Get the setting for the default postproc memory + echo "Enter the default value for the memory requested for post-processing jobs [$DEFAULT_POSTPROC_MEMORY]:" + read -p "> " input + if [ ! -z "$input" ]; then + DEFAULT_POSTPROC_MEMORY=$input + fi + echo + echo "Default post-processing memory requested is $DEFAULT_POSTPROC_MEMORY" + echo -# Get the setting for the default partition -echo "Enter the default value for the partition for simulation jobs [$DEFAULT_PARTITION]:" -read -p "> " input -if [ ! -z "$input" ]; then - DEFAULT_PARTITION=$input -fi -echo -echo "Default partiion for simulations is $DEFAULT_PARTITION" -echo + # Get the setting for the default partition + echo "Enter the default value for the partition for simulation jobs [$DEFAULT_PARTITION]:" + read -p "> " input + if [ ! -z "$input" ]; then + DEFAULT_PARTITION=$input + fi + echo + echo "Default partiion for simulations is $DEFAULT_PARTITION" + echo -# Get the setting for the default qos -echo "Enter the default value for the QOS for simulation jobs [$DEFAULT_QOS]:" -read -p "> " input -if [ ! -z "$input" ]; then - DEFAULT_QOS=$input + # Get the setting for the default qos + echo "Enter the default value for the QOS for simulation jobs [$DEFAULT_QOS]:" + read -p "> " input + if [ ! -z "$input" ]; then + DEFAULT_QOS=$input + fi + echo + echo "Default QOS for simulations is $DEFAULT_QOS" + echo fi -echo -echo "Default QOS for simulations is $DEFAULT_QOS" -echo # Now we have a 'julia' executable and the settings, can call a Julia script # (machines/shared/machine_setup.jl) to create LocalPreferences.toml, # julia.env, bin/julia, and some machine-specific symlinks. echo "Doing initial setup" echo -# Pass JULIA_DEPOT_PATH explicitly here because this script creates julia.env, -# so we cannot source it before running this script. -JULIA_DEPOT_PATH=$JULIA_DIRECTORY $JULIA machines/shared/machine_setup.jl "$MACHINE" "$ACCOUNT" "$JULIA_DIRECTORY" "$DEFAULT_RUN_TIME" "$DEFAULT_NODES" "$DEFAULT_POSTPROC_TIME" "$DEFAULT_POSTPROC_MEMORY" "$DEFAULT_PARTITION" "$DEFAULT_QOS" - -# Set up modules, JULIA_DEPOT_PATH, etc. to use for the rest of this script -source julia.env - -# Create a Python venv, ensure it contains matplotlib, and append its -# activation command to julia.env. -# Use the `--system-site-packages` option to let the venv include any packages -# already installed by the system. -PYTHON_VENV_PATH=$PWD/machines/artifacts/mk_venv -python -m venv --system-site-packages $PYTHON_VENV_PATH -source $PYTHON_VENV_PATH/bin/activate -# Use 'PYTHONNOUSERSITE=1' so that pip ignores any packages in ~/.local (which -# may not be accessible from compute nodes on some clusters) and therefore -# definitely installs matplotlib and its dependencies into mk_venv. -PYTHONNOUSERSITE=1 pip install matplotlib -echo "source $PYTHON_VENV_PATH/bin/activate" >> julia.env +# Pass JULIA_DEPOT_PATH explicitly here because this script creates bin/julia +# or julia.env, so we cannot use either before running this script. +# export JULIA_DEPOT_PATH instead of just passing as a prefix to the julia +# command, because passing as a prefix does not work (sometimes??) within a +# bash script (even though as far as JTO knows it should work). +export JULIA_DEPOT_PATH=$JULIA_DIRECTORY +$JULIA machines/shared/machine_setup.jl "$MACHINE" "$ACCOUNT" "$JULIA_DIRECTORY" "$DEFAULT_RUN_TIME" "$DEFAULT_NODES" "$DEFAULT_POSTPROC_TIME" "$DEFAULT_POSTPROC_MEMORY" "$DEFAULT_PARTITION" "$DEFAULT_QOS" + +if [ -f julia.env ]; then + # Set up modules, JULIA_DEPOT_PATH, etc. to use for the rest of this script + source julia.env +fi + +if [[ $USE_PLOTS_POSTPROC -eq 0 ]]; then + # Create a Python venv, ensure it contains matplotlib, and append its + # activation command to julia.env. + # Use the `--system-site-packages` option to let the venv include any packages + # already installed by the system. + PYTHON_VENV_PATH=$PWD/machines/artifacts/mk_venv + python -m venv --system-site-packages $PYTHON_VENV_PATH + source $PYTHON_VENV_PATH/bin/activate + # Use 'PYTHONNOUSERSITE=1' so that pip ignores any packages in ~/.local (which + # may not be accessible from compute nodes on some clusters) and therefore + # definitely installs matplotlib and its dependencies into mk_venv. + PYTHONNOUSERSITE=1 pip install matplotlib + if [[ $BATCH_SYSTEM -eq 0 ]]; then + echo "source $PYTHON_VENV_PATH/bin/activate" >> julia.env + else + # Re-write bin/julia to add activation of the Python venv + LAST_LINES=$(tail -n 2 bin/julia) + echo '#!/bin/bash' > bin/julia # It is necessary to use single-quotes not double quotes here, but don't understand why + echo "source $PYTHON_VENV_PATH/bin/activate" >> bin/julia + echo "$LAST_LINES" >> bin/julia + fi +fi # [ -f ] tests if exists and is a file if [ -f machines/shared/compile_dependencies.sh ]; then # Need to compile some dependencies echo - echo "Compliing dependencies" + echo "Compiling dependencies" machines/shared/compile_dependencies.sh fi +# We want to always add a couple of dependencies that are required to run the +# tests in the top-level environment by just 'include()'ing the test scripts. +# We also run the 'stage two' setup now if it is required. +SETUP_COMMAND="bin/julia --project -e 'import Pkg; Pkg.add([\"HDF5\", \"MPI\", \"MPIPreferences\", \"SpecialFunctions\"])" # [ -f ] tests if exists and is a file if [ -f machines/shared/machine_setup_stage_two.jl ]; then # A second setup stage exists, so run it. # This does setup for HDF5 and MPI, possibly other things if necessary. echo - echo "Doing stage two setup" - bin/julia --project machines/shared/machine_setup_stage_two.jl + echo "Including stage two setup" + SETUP_COMMAND="$SETUP_COMMAND; include(\"machines/shared/machine_setup_stage_two.jl\")" +fi +SETUP_COMMAND="$SETUP_COMMAND'" # Add the closing quote mark +eval "$SETUP_COMMAND" + +# Add moment_kinetics package to the working project +bin/julia --project -e 'import Pkg; Pkg.develop(path="moment_kinetics"); Pkg.precompile()' + +if [[ $USE_MAKIE_POSTPROC -eq 0 ]]; then + echo "Setting up makie_post_processing" + if [[ $BATCH_SYSTEM -eq 0 ]]; then + bin/julia --project=makie_post_processing/ -e 'import Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.precompile()' + else + bin/julia --project -e 'import Pkg; Pkg.develop(path="makie_post_processing")' + fi +else + if [[ $BATCH_SYSTEM -eq 1 ]]; then + bin/julia --project -e 'import Pkg; try Pkg.rm("makie_post_processing") catch end' + fi fi -echo "Do you want to submit a serial (or debug) job to precompile, creating the" -echo "moment_kinetics.so image (this is required in order to use the job submission" -echo "scripts and templates provided)? [y]/n:" -read -p "> " input +if [[ $USE_PLOTS_POSTPROC -eq 0 ]]; then + echo "Setting up plots_post_processing" + if [[ $BATCH_SYSTEM -eq 0 ]]; then + bin/julia --project=plots_post_processing/ -e 'import Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.precompile()' + else + bin/julia --project -e 'import Pkg; Pkg.develop(path="plots_post_processing")' + fi +else + if [[ $BATCH_SYSTEM -eq 1 ]]; then + bin/julia --project -e 'import Pkg; try Pkg.rm("plots_post_processing") catch end' + fi +fi -while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - # $input must be empty, 'y' or 'n'. It is none of these, so ask for input - # again until we get a valid response. - echo - echo "$input is not a valid response: [y]/n" +if [[ $BATCH_SYSTEM -eq 0 ]]; then + echo "Do you want to submit a serial (or debug) job to precompile, creating the" + echo "moment_kinetics.so image (this is required in order to use the job submission" + echo "scripts and templates provided)? [y]/n:" read -p "> " input -done -if [[ -z $input || $input == "y" ]]; then - # This script launches a job that runs precompile.jl to create the - # moment_kinetics.so image. - ./precompile-submit.sh + + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + # $input must be empty, 'y' or 'n'. It is none of these, so ask for input + # again until we get a valid response. + echo + echo "$input is not a valid response: [y]/n" + read -p "> " input + done + if [[ -z $input || $input == "y" ]]; then + # This script launches a job that runs precompile.jl to create the + # moment_kinetics.so image. + ./precompile-submit.sh + + if [[ $USE_MAKIE_POSTPROC -eq 0 ]]; then + ./precompile-makie-post-processing-submit.sh + fi + if [[ $USE_PLOTS_POSTPROC -eq 0 ]]; then + ./precompile-plots-post-processing-submit.sh + fi + fi fi echo echo "Finished!" -echo "Now run \`source julia.env\` to set up your environment, and/or add it to your .bashrc" +if [[ $BATCH_SYSTEM -eq 0 ]]; then + echo "Now run \`source julia.env\` to set up your environment, and/or add it to your .bashrc" +fi exit 0 diff --git a/machines/marconi/compile_dependencies.sh b/machines/marconi/compile_dependencies.sh index 3bd760726..55e4cbe8e 100755 --- a/machines/marconi/compile_dependencies.sh +++ b/machines/marconi/compile_dependencies.sh @@ -11,7 +11,7 @@ ARTIFACT_DIR=$PWD BUILDHDF5=0 if [ -d hdf5-build ]; then echo "HDF5 appears to have been downloaded, compiled and installed already." - echo "Do you want to download, compile and install again, overwriting the exsiting " + echo "Do you want to download, compile and install again, overwriting the existing " echo "version? y/[n]" read -p "> " input while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do diff --git a/machines/marconi/jobscript-postprocess-plotsjl.template b/machines/marconi/jobscript-postprocess-plotsjl.template index 93edb12c4..af338535d 100644 --- a/machines/marconi/jobscript-postprocess-plotsjl.template +++ b/machines/marconi/jobscript-postprocess-plotsjl.template @@ -14,6 +14,6 @@ cd $SLURM_SUBMIT_DIR source julia.env echo "post-processing (with original post_processing) RUNDIR $(date)" -bin/julia -Jmoment_kinetics.so --project run_post_processing.jl RUNDIR +bin/julia -Jplots_postproc.so --project=plots_post_processing/ run_post_processing.jl RUNDIR echo "finished post-processing RUNDIR $(date)" diff --git a/machines/marconi/jobscript-postprocess.template b/machines/marconi/jobscript-postprocess.template index 1ab7b1af9..c6f2e942e 100644 --- a/machines/marconi/jobscript-postprocess.template +++ b/machines/marconi/jobscript-postprocess.template @@ -14,6 +14,6 @@ cd $SLURM_SUBMIT_DIR source julia.env echo "post-processing RUNDIR $(date)" -bin/julia -Jmakie_postproc.so --project run_makie_post_processing.jl RUNDIR +bin/julia -Jmakie_postproc.so --project=makie_post_processing/ run_makie_post_processing.jl RUNDIR echo "finished post-processing RUNDIR $(date)" diff --git a/machines/marconi/jobscript-precompile-postprocessing.template b/machines/marconi/jobscript-precompile-makie-post-processing.template similarity index 77% rename from machines/marconi/jobscript-precompile-postprocessing.template rename to machines/marconi/jobscript-precompile-makie-post-processing.template index aaf7658e6..75423540c 100644 --- a/machines/marconi/jobscript-precompile-postprocessing.template +++ b/machines/marconi/jobscript-precompile-makie-post-processing.template @@ -15,6 +15,6 @@ source julia.env echo "precompiling $(date)" -bin/julia --project precompile-makie-post-processing.jl +bin/julia --project=makie_post_processing/ precompile-makie-post-processing.jl echo "finished!" diff --git a/machines/marconi/jobscript-precompile-plots-post-processing.template b/machines/marconi/jobscript-precompile-plots-post-processing.template new file mode 100644 index 000000000..5234137fd --- /dev/null +++ b/machines/marconi/jobscript-precompile-plots-post-processing.template @@ -0,0 +1,20 @@ +#!/bin/bash + +#SBATCH --ntasks=1 +#SBATCH --time=1:00:00 +#SBATCH --account=ACCOUNT +#SBATCH --partition=skl_fua_dbg +#SBATCH --output=PRECOMPILEDIRslurm-%j.out + +set -e + +cd $SLURM_SUBMIT_DIR + +# Get setup for Julia +source julia.env + +echo "precompiling $(date)" + +bin/julia --project=plots_post_processing/ precompile-plots-post-processing.jl + +echo "finished!" diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index 6f7fa19b0..40ffd24b0 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -16,6 +16,12 @@ default_settings["base"] = Dict("default_run_time"=>"24:00:00", "default_postproc_memory"=>"64G", "default_partition"=>"", "default_qos"=>"") +# No batch system steup for "generic-pc" +default_settings["generic-pc"] = merge(default_settings["base"], + Dict("default_run_time"=>"0:00:00", + "default_nodes"=>"0", + "default_postproc_time"=>"0:00:00", + "default_postproc_memory"=>"0")) default_settings["archer"] = merge(default_settings["base"], Dict("default_partition"=>"standard", "default_qos"=>"standard")) @@ -62,8 +68,8 @@ Do setup for a known `machine`: `julia_directory` gives the location of the directory (usually called `.julia`) where Julia installs files, saves settings, etc. `julia_directory` must be passed if this directory should be in a non-default location (i.e. not `\$HOME/.julia/`). The value is -used to set `JULIA_DEPOT_PATH` in the `julia.env` file, so that this setting is -propagated to the environment on the compute nodes. +used to set `JULIA_DEPOT_PATH` in the `julia.env` file or `bin/julia` script , so that +this setting is propagated to the environment on the compute nodes. Usually it is necessary for Julia to be restarted after running this function to ensure the correct MPI is linked, etc. so the function will force Julia to exit. If for some @@ -95,6 +101,8 @@ re-running this function (if you want to). The arguments are: change this to a free, low-priority queue if one is available. Currently supported machines: +* `"generic-pc"` - A generic personal computer. Set up for interactive use, rather than + for submitting jobs to a batch queue. * `"archer"` - the UK supercomputer [ARCHER2](https://www.archer2.ac.uk/) * `"marconi"` - the EUROfusion supercomputer [Marconi](https://wiki.u-gov.it/confluence/display/SCAIUS/UG3.1%3A+MARCONI+UserGuide) @@ -121,27 +129,66 @@ function machine_setup_moment_kinetics(machine::String, # Common operations that only depend on the name of `machine` ############################################################# - println("\n** Creating `julia.env` for environment setup\n") - envname = joinpath(repo_dir, "julia.env") - # Read the template - template = read("machines/$machine/julia.env") - # Write julia.env, overwriting if it already exists - ispath(envname) && rm(envname) - open(envname, "w") do io - write(io, template) - if julia_directory != "" - println("\n** Setting JULIA_DEPOT_PATH=$julia_directory in `julia.env`\n") - println(io, "\nexport JULIA_DEPOT_PATH=$julia_directory") + if machine == "generic-pc" + batch_system = false + else + batch_system = true + end + + if batch_system + # Only use julia.env for a batch system as for an interactive system there are no + # modules to set up and source'ing julia.env is mildly inconvenient. + println("\n** Creating `julia.env` for environment setup\n") + envname = joinpath(repo_dir, "julia.env") + # Read the template + template = read("machines/$machine/julia.env") + # Write julia.env, overwriting if it already exists + ispath(envname) && rm(envname) + open(envname, "w") do io + write(io, template) + if julia_directory != "" + println("\n** Setting JULIA_DEPOT_PATH=$julia_directory in `julia.env`\n") + println(io, "\nexport JULIA_DEPOT_PATH=$julia_directory") + end end end - # Make a local link to the Julia binary so scripts in the repo can find it - println("\n** Making a symlink to the julia executable at bin/julia\n") bindir = joinpath(repo_dir, "bin") mkpath(bindir) julia_executable_name = joinpath(bindir, "julia") - islink(julia_executable_name) && rm(julia_executable_name) - symlink(joinpath(Sys.BINDIR, "julia"), julia_executable_name) + if batch_system || julia_directory == "" + # Make a local link to the Julia binary so scripts in the repo can find it + println("\n** Making a symlink to the julia executable at bin/julia\n") + islink(julia_executable_name) && rm(julia_executable_name) + symlink(joinpath(Sys.BINDIR, "julia"), julia_executable_name) + else + # Make a script to run julia, including the JULIA_DEPOT_PATH so that we can avoid + # needing the julia.env setup + open(julia_executable_name, "w") do io + println(io, "#!/bin/bash") + println(io, "export JULIA_DEPOT_PATH=$julia_directory") + julia_path = joinpath(Sys.BINDIR, "julia") + println(io, "$julia_path \"\$@\"") + end + function make_executable!(file) + user_permissions = uperm(file) + group_permissions = gperm(file) + other_permissions = operm(file) + + # Change each permissions field to be executable + user_permissions = user_permissions | 0x01 + group_permissions = group_permissions | 0x01 + other_permissions = other_permissions | 0x01 + + permissions = user_permissions * UInt16(0o100) + + group_permissions * UInt16(0o10) + + other_permissions * UInt16(0o1) + chmod(file, permissions) + + return nothing + end + make_executable!(julia_executable_name) + end # Write these preferences into a [moment_kinetics] section in LocalPreferences.toml # @@ -189,7 +236,13 @@ function machine_setup_moment_kinetics(machine::String, # non-empty `account` setting needs_account = false - if machine == "archer" + if machine == "generic-pc" + # For generic-pc, run compile_dependencies.sh script to optionally download and + # compile HDF5 + needs_compile_dependencies = true + + needs_second_stage = true + elseif machine == "archer" needs_account = true if julia_directory == "" error("On ARCHER2, the `julia_directory` setting is required, because the " @@ -243,9 +296,9 @@ function machine_setup_moment_kinetics(machine::String, if interactive println() println("***********************************************************************") - println("To complete setup, first `source julia.env` (so that `JULIA_DEPOT_PATH`") - println("is set correctly, then start Julia again (you can now use ") - println("`bin/julia --project`) and to complete the setup run:") + println("To complete setup, first `source julia.env` if it exists (so that") + println("`JULIA_DEPOT_PATH` is set correctly), then start Julia again (you can") + println("now use `bin/julia --project`) and to complete the setup run:") println(" julia> include(\"$second_stage_relative_path\")") println("***********************************************************************") end diff --git a/precompile-makie-post-processing-submit.sh b/precompile-makie-post-processing-submit.sh new file mode 100755 index 000000000..b40fbd093 --- /dev/null +++ b/precompile-makie-post-processing-submit.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -e + +# Parse command line options +while getopts "h" opt; do + case $opt in + h) + echo "Submit job to precompile moment kinetics +-h Print help and exit" + exit 1 + ;; + esac +done + +# job settings for the run +########################## + +# Get setup for Julia +source julia.env + +JOBINFO=($(util/get-precompile-info.jl)) +MACHINE=${JOBINFO[0]} +ACCOUNT=${JOBINFO[1]} + +PRECOMPILEDIR=precompile-temp/ +mkdir -p $PRECOMPILEDIR + +# Create a submission script for the post-processing precompilation +POSTPROCESSINGJOBSCRIPT=${PRECOMPILEDIR}precompile-makie-post-processing.job +sed -e "s|ACCOUNT|$ACCOUNT|" -e "s|PRECOMPILEDIR|$PRECOMPILEDIR|" machines/$MACHINE/jobscript-precompile-makie-post-processing.template > $POSTPROCESSINGJOBSCRIPT + +JOBID=$(sbatch --parsable $POSTPROCESSINGJOBSCRIPT) +echo "Precompile makie_post_processing: $JOBID" +echo "In the queue" > $PRECOMPILEDIR/slurm-$JOBID.out + +echo "Done" diff --git a/precompile-makie-post-processing.jl b/precompile-makie-post-processing.jl index 1afe4d4df..41674db91 100644 --- a/precompile-makie-post-processing.jl +++ b/precompile-makie-post-processing.jl @@ -1,8 +1,5 @@ using Pkg -# Activate the moment_kinetics package -Pkg.activate(".") - using PackageCompiler # Create the sysimage 'makie_postproc.so' in the base moment_kinetics source directory diff --git a/precompile-plots-post-processing-submit.sh b/precompile-plots-post-processing-submit.sh new file mode 100755 index 000000000..d762b9131 --- /dev/null +++ b/precompile-plots-post-processing-submit.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -e + +# Parse command line options +while getopts "h" opt; do + case $opt in + h) + echo "Submit job to precompile moment kinetics +-h Print help and exit" + exit 1 + ;; + esac +done + +# job settings for the run +########################## + +# Get setup for Julia +source julia.env + +JOBINFO=($(util/get-precompile-info.jl)) +MACHINE=${JOBINFO[0]} +ACCOUNT=${JOBINFO[1]} + +PRECOMPILEDIR=precompile-temp/ +mkdir -p $PRECOMPILEDIR + +# Create a submission script for the post-processing precompilation +POSTPROCESSINGJOBSCRIPT=${PRECOMPILEDIR}precompile-plots-post-processing.job +sed -e "s|ACCOUNT|$ACCOUNT|" -e "s|PRECOMPILEDIR|$PRECOMPILEDIR|" machines/$MACHINE/jobscript-precompile-plots-post-processing.template > $POSTPROCESSINGJOBSCRIPT + +JOBID=$(sbatch --parsable $POSTPROCESSINGJOBSCRIPT) +echo "Precompile plots_post_processing: $JOBID" +echo "In the queue" > $PRECOMPILEDIR/slurm-$JOBID.out + +echo "Done" diff --git a/precompile-plots-post-processing.jl b/precompile-plots-post-processing.jl new file mode 100644 index 000000000..a2887282e --- /dev/null +++ b/precompile-plots-post-processing.jl @@ -0,0 +1,13 @@ +using Pkg + +using PackageCompiler + +# Create the sysimage 'makie_postproc.so' in the base moment_kinetics source directory +# with both moment_kinetics and the dependencies listed above precompiled. +# Warning: editing the code will not affect what runs when using this .so, you +# need to re-precompile if you change anything. +create_sysimage(; sysimage_path="plots_postproc.so", + precompile_execution_file="util/precompile_plots_plots.jl", + include_transitive_dependencies=false, # This is needed to make MPI work, see https://github.com/JuliaParallel/MPI.jl/issues/518 + sysimage_build_args=`-O3`, # Assume if we are precompiling we want an optimized, production build. No `--check-bounds=no` because Makie doesn' like it https://github.com/MakieOrg/Makie.jl/issues/3132 + ) diff --git a/precompile-submit.sh b/precompile-submit.sh index c0fd3fd78..873ceb453 100755 --- a/precompile-submit.sh +++ b/precompile-submit.sh @@ -46,12 +46,4 @@ JOBID=$(sbatch --parsable $JOBSCRIPT) echo "Precompile: $JOBID" echo "In the queue" > $PRECOMPILEDIR/slurm-$JOBID.out -# Create a submission script for the post-processing precompilation -POSTPROCESSINGJOBSCRIPT=${PRECOMPILEDIR}precompile-postprocessing.job -sed -e "s|ACCOUNT|$ACCOUNT|" -e "s|PRECOMPILEDIR|$PRECOMPILEDIR|" machines/$MACHINE/jobscript-precompile-postprocessing.template > $POSTPROCESSINGJOBSCRIPT - -JOBID=$(sbatch --parsable $POSTPROCESSINGJOBSCRIPT) -echo "Precompile postprocessing: $JOBID" -echo "In the queue" > $PRECOMPILEDIR/slurm-$JOBID.out - echo "Done" diff --git a/run_makie_post_processing.jl b/run_makie_post_processing.jl index 8853722fa..2a6fc2aa6 100644 --- a/run_makie_post_processing.jl +++ b/run_makie_post_processing.jl @@ -1,8 +1,2 @@ -if abspath(PROGRAM_FILE) == @__FILE__ - using Pkg - Pkg.activate(".") - - using moment_kinetics.makie_post_processing - - makie_post_process(ARGS...) -end +using makie_post_processing +makie_post_process(ARGS...) diff --git a/run_post_processing.jl b/run_post_processing.jl index 81e8d4282..4f43ae6e1 100644 --- a/run_post_processing.jl +++ b/run_post_processing.jl @@ -1,8 +1,2 @@ -if abspath(PROGRAM_FILE) == @__FILE__ - using Pkg - Pkg.activate(".") - - import moment_kinetics as mk - - mk.post_processing.analyze_and_plot_data(ARGS...) -end +using plots_post_processing +plots_post_processing.analyze_and_plot_data(ARGS...) diff --git a/util/precompile_makie_plots.jl b/util/precompile_makie_plots.jl index 84cd86b67..6d9b48ff0 100644 --- a/util/precompile_makie_plots.jl +++ b/util/precompile_makie_plots.jl @@ -2,7 +2,7 @@ using Pkg Pkg.activate(".") -using moment_kinetics +using makie_post_processing # Create a temporary directory for test output test_output_directory = tempname() @@ -59,5 +59,5 @@ for (k,v) ∈ precompile_postproc_options end end -moment_kinetics.makie_post_processing.makie_post_process( - joinpath(test_output_directory, run_name), precompile_postproc_options) +makie_post_processing.makie_post_process(joinpath(test_output_directory, run_name), + precompile_postproc_options) From b20181ce686ed4e27f4f6a9feb501de8974355fb Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 21 Dec 2023 11:04:21 +0000 Subject: [PATCH 20/80] Add .julia to the .gitignore This is convenient when the user chooses to have their .julia locally in the project directory. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0f458cf2a..292b32c93 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ Manifest.toml LocalPreferences.toml /Project.toml /julia.env +/.julia bin machines/shared/compile_dependencies.sh machines/shared/machine_setup_stage_two.jl From 5c05a68228a47f78b28386a39882998b61a8b599 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 21 Dec 2023 13:14:41 +0000 Subject: [PATCH 21/80] Only provide submission scripts on batch systems When not on a cluster with a batch job submission system, the shell scripts to submit jobs are just clutter in the top level directory. Move the scripts to machines/shared, and symlink them at the top level when we set up moment_kinetics for a batch system. --- .gitignore | 5 +++++ machines/machine_setup.sh | 13 +++++++++++++ .../precompile-makie-post-processing-submit.sh | 0 .../precompile-plots-post-processing-submit.sh | 0 .../shared/precompile-submit.sh | 0 .../shared/submit-restart.sh | 0 submit-run.sh => machines/shared/submit-run.sh | 0 7 files changed, 18 insertions(+) rename precompile-makie-post-processing-submit.sh => machines/shared/precompile-makie-post-processing-submit.sh (100%) rename precompile-plots-post-processing-submit.sh => machines/shared/precompile-plots-post-processing-submit.sh (100%) rename precompile-submit.sh => machines/shared/precompile-submit.sh (100%) rename submit-restart.sh => machines/shared/submit-restart.sh (100%) rename submit-run.sh => machines/shared/submit-run.sh (100%) diff --git a/.gitignore b/.gitignore index 292b32c93..73d1414cb 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,11 @@ bin machines/shared/compile_dependencies.sh machines/shared/machine_setup_stage_two.jl machines/artifacts +/precompile-makie-post-processing-submit.sh +/precompile-plots-post-processing-submit.sh +/precompile-submit.sh +/submit-restart.sh +/submit-run.sh moment_kinetics.so precompile-temp post_processing_input.toml diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 4d9c9102c..06b4b5fce 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -428,6 +428,19 @@ else fi fi +if [[ $BATCH_SYSTEM -eq 0 ]]; then + # Make symlinks to submission scripts + ln -s machines/shared/precompile-submit.sh + ln -s machines/shared/submit-run.sh + ln -s machines/shared/submit-restart.sh + if [[ $USE_MAKIE_POSTPROC -eq 0 ]]; then + ln -s machines/shared/precompile-makie-post-processing-submit.sh + fi + if [[ $USE_PLOTS_POSTPROC -eq 0 ]]; then + ln -s machines/shared/precompile-plots-post-processing-submit.sh + fi +fi + if [[ $BATCH_SYSTEM -eq 0 ]]; then echo "Do you want to submit a serial (or debug) job to precompile, creating the" echo "moment_kinetics.so image (this is required in order to use the job submission" diff --git a/precompile-makie-post-processing-submit.sh b/machines/shared/precompile-makie-post-processing-submit.sh similarity index 100% rename from precompile-makie-post-processing-submit.sh rename to machines/shared/precompile-makie-post-processing-submit.sh diff --git a/precompile-plots-post-processing-submit.sh b/machines/shared/precompile-plots-post-processing-submit.sh similarity index 100% rename from precompile-plots-post-processing-submit.sh rename to machines/shared/precompile-plots-post-processing-submit.sh diff --git a/precompile-submit.sh b/machines/shared/precompile-submit.sh similarity index 100% rename from precompile-submit.sh rename to machines/shared/precompile-submit.sh diff --git a/submit-restart.sh b/machines/shared/submit-restart.sh similarity index 100% rename from submit-restart.sh rename to machines/shared/submit-restart.sh diff --git a/submit-run.sh b/machines/shared/submit-run.sh similarity index 100% rename from submit-run.sh rename to machines/shared/submit-run.sh From 1c88e6f2cd0fcc323f347cb803edf1ab598939c0 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 21 Dec 2023 13:30:20 +0000 Subject: [PATCH 22/80] Move prompt for submitting precompilation near beginning of script This change puts all the prompts where interactive user input is required to the same place, near the beginning of the script, so that the user can go away and do other things while (potentially) slow parts like precompilation are running. --- machines/machine_setup.sh | 54 ++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 06b4b5fce..417b04926 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -335,6 +335,28 @@ if [[ $BATCH_SYSTEM -eq 0 ]]; then echo fi +if [[ $BATCH_SYSTEM -eq 0 ]]; then + echo "Do you want to submit a serial (or debug) job to precompile, creating the" + echo "moment_kinetics.so image (this is required in order to use the job submission" + echo "scripts and templates provided)? [y]/n:" + read -p "> " input + + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + # $input must be empty, 'y' or 'n'. It is none of these, so ask for input + # again until we get a valid response. + echo + echo "$input is not a valid response: [y]/n" + read -p "> " input + done + if [[ -z $input || $input == "y" ]]; then + SUBMIT_PRECOMPILATION=0 + else + SUBMIT_PRECOMPILATION=1 + fi +else + SUBMIT_PRECOMPILATION=1 +fi + # Now we have a 'julia' executable and the settings, can call a Julia script # (machines/shared/machine_setup.jl) to create LocalPreferences.toml, # julia.env, bin/julia, and some machine-specific symlinks. @@ -441,30 +463,16 @@ if [[ $BATCH_SYSTEM -eq 0 ]]; then fi fi -if [[ $BATCH_SYSTEM -eq 0 ]]; then - echo "Do you want to submit a serial (or debug) job to precompile, creating the" - echo "moment_kinetics.so image (this is required in order to use the job submission" - echo "scripts and templates provided)? [y]/n:" - read -p "> " input +if [[ $SUBMIT_PRECOMPILATION -eq 0 ]]; then + # This script launches a job that runs precompile.jl to create the + # moment_kinetics.so image. + ./precompile-submit.sh - while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - # $input must be empty, 'y' or 'n'. It is none of these, so ask for input - # again until we get a valid response. - echo - echo "$input is not a valid response: [y]/n" - read -p "> " input - done - if [[ -z $input || $input == "y" ]]; then - # This script launches a job that runs precompile.jl to create the - # moment_kinetics.so image. - ./precompile-submit.sh - - if [[ $USE_MAKIE_POSTPROC -eq 0 ]]; then - ./precompile-makie-post-processing-submit.sh - fi - if [[ $USE_PLOTS_POSTPROC -eq 0 ]]; then - ./precompile-plots-post-processing-submit.sh - fi + if [[ $USE_MAKIE_POSTPROC -eq 0 ]]; then + ./precompile-makie-post-processing-submit.sh + fi + if [[ $USE_PLOTS_POSTPROC -eq 0 ]]; then + ./precompile-plots-post-processing-submit.sh fi fi From 97f03b6fd960746bcf73097018378ad36ab45226 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 21 Dec 2023 13:44:20 +0000 Subject: [PATCH 23/80] Use previous settings as defaults when machine_setup.sh is re-run --- .gitignore | 1 + machines/machine_setup.sh | 143 +++++++++++++++++++++---------- machines/shared/machine_setup.jl | 54 ++++++++++-- 3 files changed, 145 insertions(+), 53 deletions(-) diff --git a/.gitignore b/.gitignore index 73d1414cb..7d781422c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ machines/artifacts /precompile-submit.sh /submit-restart.sh /submit-run.sh +/.this_machine_name.txt moment_kinetics.so precompile-temp post_processing_input.toml diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 417b04926..91bd91016 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -32,15 +32,19 @@ touch Project.toml # [See https://stackoverflow.com/a/13400237] JULIA=${@:$OPTIND:1} -# Apply heuristics to try and get a default value for MACHINE. -# Note these are machine-specific guesses, and the tests may well be broken by -# changes to machine configuration, etc. -if ls /marconi > /dev/null 2>&1; then - DEFAULT_MACHINE=marconi -elif module avail 2>&1 | grep -q epcc; then - DEFAULT_MACHINE=archer +if [[ -f .this_machine_name.txt ]]; then + DEFAULT_MACHINE=$(cat .this_machine_name.txt) else - DEFAULT_MACHINE=generic-pc + # Apply heuristics to try and get a default value for MACHINE. + # Note these are machine-specific guesses, and the tests may well be broken by + # changes to machine configuration, etc. + if ls /marconi > /dev/null 2>&1; then + DEFAULT_MACHINE=marconi + elif module avail 2>&1 | grep -q epcc; then + DEFAULT_MACHINE=archer + else + DEFAULT_MACHINE=generic-pc + fi fi # Create directory to set up Python venv and download/compile dependencies in @@ -182,46 +186,16 @@ done echo "Setting up for '$MACHINE'" echo +# Save the machine name so we can use it as the default if we re-run +# machine_setup.sh. +echo $MACHINE > .this_machine_name.txt + if [[ $MACHINE -eq "generic-pc" ]]; then BATCH_SYSTEM=1 else BATCH_SYSTEM=0 fi -echo "Would you like to set up makie_post_processing? y/[n]" -read -p "> " input -echo -while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - # $input must be empty, 'y' or 'n'. It is none of these, so ask for input - # again until we get a valid response. - echo - echo "$input is not a valid response: y/[n]" - read -p "> " input - echo -done -if [[ $input == "y" ]]; then - USE_MAKIE_POSTPROC=0 -else - USE_MAKIE_POSTPROC=1 -fi - -echo "Would you like to set up plots_post_processing? y/[n]" -read -p "> " input -echo -while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - # $input must be empty, 'y' or 'n'. It is none of these, so ask for input - # again until we get a valid response. - echo - echo "$input is not a valid response: y/[n]" - read -p "> " input - echo -done -if [[ $input == "y" ]]; then - USE_PLOTS_POSTPROC=0 -else - USE_PLOTS_POSTPROC=1 -fi - # Note that the $(eval echo )) is needed to remove quotes around # arguments. Adding the quotes in the Julia script is necessary so that if an # argument is empty it is not lost when parsing the Julia script output into @@ -232,11 +206,90 @@ DEFAULT_POSTPROC_TIME=$(eval echo ${DEFAULTS[2]}) DEFAULT_POSTPROC_MEMORY=$(eval echo ${DEFAULTS[3]}) DEFAULT_PARTITION=$(eval echo ${DEFAULTS[4]}) DEFAULT_QOS=$(eval echo ${DEFAULTS[5]}) +DEFAULT_ACCOUNT=$(eval echo ${DEFAULTS[6]}) +JULIA_DIRECTORY=$(eval echo ${DEFAULTS[7]}) +DEFAULT_USE_MAKIE_POSTPROC=$(eval echo ${DEFAULTS[8]}) +DEFAULT_USE_PLOTS_POSTPROC=$(eval echo ${DEFAULTS[9]}) + +if [[ $DEFAULT_USE_MAKIE_POSTPROC -eq 0 ]]; then + echo "Would you like to set up makie_post_processing? [y]/n" + read -p "> " input + echo + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + # $input must be empty, 'y' or 'n'. It is none of these, so ask for input + # again until we get a valid response. + echo + echo "$input is not a valid response: [y]/n" + read -p "> " input + echo + done + if [[ $input == "n" ]]; then + USE_MAKIE_POSTPROC=1 + else + USE_MAKIE_POSTPROC=0 + fi +else + echo "Would you like to set up makie_post_processing? y/[n]" + read -p "> " input + echo + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + # $input must be empty, 'y' or 'n'. It is none of these, so ask for input + # again until we get a valid response. + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + echo + done + if [[ $input == "y" ]]; then + USE_MAKIE_POSTPROC=0 + else + USE_MAKIE_POSTPROC=1 + fi +fi + +if [[ $DEFAULT_USE_PLOTS_POSTPROC -eq 0 ]]; then + echo "Would you like to set up plots_post_processing? [y]/n" + read -p "> " input + echo + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + # $input must be empty, 'y' or 'n'. It is none of these, so ask for input + # again until we get a valid response. + echo + echo "$input is not a valid response: [y]/n" + read -p "> " input + echo + done + if [[ $input == "n" ]]; then + USE_PLOTS_POSTPROC=1 + else + USE_PLOTS_POSTPROC=0 + fi +else + echo "Would you like to set up plots_post_processing? y/[n]" + read -p "> " input + echo + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + # $input must be empty, 'y' or 'n'. It is none of these, so ask for input + # again until we get a valid response. + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + echo + done + if [[ $input == "y" ]]; then + USE_PLOTS_POSTPROC=0 + else + USE_PLOTS_POSTPROC=1 + fi +fi if [[ $BATCH_SYSTEM -eq 0 ]]; then # Get the account to submit jobs with - echo "Enter the account code used to submit jobs []:" + echo "Enter the account code used to submit jobs [$DEFAULT_ACCOUNT]:" read -p "> " ACCOUNT + if [[ -z ACCOUNT ]]; then + ACCOUNT=$DEFAULT_ACCOUNT + fi echo echo "Account code used is $ACCOUNT" echo @@ -247,8 +300,6 @@ fi # Get the location for the .julia directory, in case this has to have a # non-default value, e.g. because the user's home directory is not accessible # from compute nodes. -# Use $JULIA_DEPOT_PATH as the default if it has already been set -JULIA_DIRECTORY=$JULIA_DEPOT_PATH echo "It can be useful or necessary to set a non-default location for the " echo ".julia directory. Leave this empty if the default location is OK." echo "Enter a name for a subdirectory of the current directory, e.g. " @@ -368,7 +419,7 @@ echo # command, because passing as a prefix does not work (sometimes??) within a # bash script (even though as far as JTO knows it should work). export JULIA_DEPOT_PATH=$JULIA_DIRECTORY -$JULIA machines/shared/machine_setup.jl "$MACHINE" "$ACCOUNT" "$JULIA_DIRECTORY" "$DEFAULT_RUN_TIME" "$DEFAULT_NODES" "$DEFAULT_POSTPROC_TIME" "$DEFAULT_POSTPROC_MEMORY" "$DEFAULT_PARTITION" "$DEFAULT_QOS" +$JULIA machines/shared/machine_setup.jl "$MACHINE" "$ACCOUNT" "$JULIA_DIRECTORY" "$DEFAULT_RUN_TIME" "$DEFAULT_NODES" "$DEFAULT_POSTPROC_TIME" "$DEFAULT_POSTPROC_MEMORY" "$DEFAULT_PARTITION" "$DEFAULT_QOS" "$USE_MAKIE_POSTPROC" "$USE_PLOTS_POSTPROC" if [ -f julia.env ]; then # Set up modules, JULIA_DEPOT_PATH, etc. to use for the rest of this script diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index 40ffd24b0..1859f17c4 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -15,13 +15,16 @@ default_settings["base"] = Dict("default_run_time"=>"24:00:00", "default_postproc_time"=>"1:00:00", "default_postproc_memory"=>"64G", "default_partition"=>"", - "default_qos"=>"") + "default_qos"=>"", + "use_makie"=>"1", + "use_plots"=>"1") # No batch system steup for "generic-pc" default_settings["generic-pc"] = merge(default_settings["base"], Dict("default_run_time"=>"0:00:00", "default_nodes"=>"0", "default_postproc_time"=>"0:00:00", - "default_postproc_memory"=>"0")) + "default_postproc_memory"=>"0", + "use_makie"=>"0")) default_settings["archer"] = merge(default_settings["base"], Dict("default_partition"=>"standard", "default_qos"=>"standard")) @@ -40,6 +43,8 @@ default_settings["marconi"] = merge(default_settings["base"], default_postproc_memory::String, default_partition::String; default_qos::String; + use_makie::String; + use_plots::String; no_force_exit::Bool=false, interactive::Bool=true) @@ -99,6 +104,10 @@ re-running this function (if you want to). The arguments are: cluster's documentation for possible values. The default will be the standard queue, which charges towards the budget of your allocation. You might want, for example, to change this to a free, low-priority queue if one is available. +* `use_makie` indicates whether makie_post_processing has been enabled ("0" means yes, "1" + means no). +* `use_plots` indicates whether plots_post_processing has been enabled ("0" means yes, "1" + means no). Currently supported machines: * `"generic-pc"` - A generic personal computer. Set up for interactive use, rather than @@ -120,7 +129,9 @@ function machine_setup_moment_kinetics(machine::String, default_postproc_time::String, default_postproc_memory::String, default_partition::String, - default_qos::String; + default_qos::String, + use_makie::String, + use_plots::String; no_force_exit::Bool=false, interactive::Bool=true) @@ -204,8 +215,8 @@ function machine_setup_moment_kinetics(machine::String, local_preferences = Dict{String,Any}() end # Always overwrite any existing preferences, to get a fresh setup - mk_preferences = local_preferences["moment_kinetics"] = Dict{String,Any}() - mk_preferences["machine"] = machine + mk_preferences = local_preferences["moment_kinetics"] = Dict{String,String}() + mk_preferences["julia_directory"] = julia_directory mk_preferences["default_run_time"] = default_run_time mk_preferences["default_nodes"] = default_nodes mk_preferences["default_postproc_time"] = default_postproc_time @@ -213,6 +224,8 @@ function machine_setup_moment_kinetics(machine::String, mk_preferences["default_partition"] = default_partition mk_preferences["default_qos"] = default_qos mk_preferences["account"] = account + mk_preferences["use_makie"] = use_makie + mk_preferences["use_plots"] = use_plots open(local_preferences_filename, "w") do io TOML.print(io, local_preferences, sorted=true) end @@ -314,6 +327,7 @@ end end using .machine_setup +using TOML if abspath(PROGRAM_FILE) == @__FILE__ # Allow the command to be called as a script. @@ -347,10 +361,36 @@ if abspath(PROGRAM_FILE) == @__FILE__ exit(1) end - d = machine_setup.default_settings[machine] + d = deepcopy(machine_setup.default_settings[machine]) + # default setting for "julia_directory" is the JULIA_DEPOT_PATH environment + # variable + d["julia_directory"] = get(ENV, "JULIA_DEPOT_PATH", "") + # No default for "account". + d["account"] = "" + + # If settings have already been saved (i.e. machine_setup.sh has already been + # run), then use the previous settings as the default this time. + # Use TOML.parsefile() to read the existing preferences to avoid depending on the + # Preferences package at this point (because we might want to set + # JULIA_DEPOT_PATH, but not have set it yet). + if ispath("LocalPreferences.toml") + existing_settings = get(TOML.parsefile("LocalPreferences.toml"), + "moment_kinetics", Dict{String, String}()) + else + existing_settings = Dict{String, String}() + end + for setting ∈ ("default_run_time", "default_nodes", "default_postproc_time", + "default_postproc_memory", "default_partition", + "default_partition", "account", "julia_directory", "use_makie", + "use_plots") + d[setting] = get(existing_settings, setting, d[setting]) + end + println("\"", d["default_run_time"], "\" \"",d["default_nodes"], "\" \"", d["default_postproc_time"], "\" \"", d["default_postproc_memory"], - "\" \"", d["default_partition"], "\" \"", d["default_qos"], "\"") + "\" \"", d["default_partition"], "\" \"", d["default_qos"], "\" \"", + d["account"], "\" \"", d["julia_directory"], "\" \"", d["use_makie"], + "\" \"", d["use_plots"], "\"") exit(0) end From 6b4d505a12318320ea6a565a3a56bb6e65f9b039 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 21 Dec 2023 16:30:31 +0000 Subject: [PATCH 24/80] Remove unused NCDatasets dependency from plots_post_processing --- plots_post_processing/Project.toml | 1 - plots_post_processing/src/plot_MMS_sequence.jl | 1 - plots_post_processing/src/plot_sequence.jl | 1 - plots_post_processing/src/plots_post_processing.jl | 1 - 4 files changed, 4 deletions(-) diff --git a/plots_post_processing/Project.toml b/plots_post_processing/Project.toml index 5d0a85d0f..6655aa304 100644 --- a/plots_post_processing/Project.toml +++ b/plots_post_processing/Project.toml @@ -10,7 +10,6 @@ LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" LsqFit = "2fda8390-95c7-5789-9bda-21331edee243" MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" Measures = "442fdcdd-2543-5da2-b0f3-8c86c306513e" -NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" PyPlot = "d330b81b-6aea-500a-939a-2ce795aea3ee" diff --git a/plots_post_processing/src/plot_MMS_sequence.jl b/plots_post_processing/src/plot_MMS_sequence.jl index 37554aa0d..579829503 100644 --- a/plots_post_processing/src/plot_MMS_sequence.jl +++ b/plots_post_processing/src/plot_MMS_sequence.jl @@ -8,7 +8,6 @@ export get_MMS_error_data using Plots using IJulia using LsqFit -using NCDatasets using Statistics: mean using SpecialFunctions: erfi using LaTeXStrings diff --git a/plots_post_processing/src/plot_sequence.jl b/plots_post_processing/src/plot_sequence.jl index c9e107026..6035b1cc2 100644 --- a/plots_post_processing/src/plot_sequence.jl +++ b/plots_post_processing/src/plot_sequence.jl @@ -6,7 +6,6 @@ module plot_sequence using Plots using IJulia using LsqFit -using NCDatasets using Statistics: mean using SpecialFunctions: erfi using LaTeXStrings diff --git a/plots_post_processing/src/plots_post_processing.jl b/plots_post_processing/src/plots_post_processing.jl index 8820cfb1f..a6a7b1667 100644 --- a/plots_post_processing/src/plots_post_processing.jl +++ b/plots_post_processing/src/plots_post_processing.jl @@ -26,7 +26,6 @@ using Plots using Glob using IJulia using LsqFit -using NCDatasets using Statistics: mean using SpecialFunctions: erfi using LaTeXStrings From c931fccf5ff3e514958c8a1a3033aa5a0a37cbf7 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 28 Dec 2023 19:22:45 +0000 Subject: [PATCH 25/80] Optionally set up NetCDF and/or MMS during machine setup --- machines/machine_setup.sh | 101 ++++++++++++++++++++++++++++++- machines/shared/machine_setup.jl | 21 +++++-- moment_kinetics/Project.toml | 3 +- 3 files changed, 118 insertions(+), 7 deletions(-) diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 91bd91016..1814822ea 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -210,6 +210,8 @@ DEFAULT_ACCOUNT=$(eval echo ${DEFAULTS[6]}) JULIA_DIRECTORY=$(eval echo ${DEFAULTS[7]}) DEFAULT_USE_MAKIE_POSTPROC=$(eval echo ${DEFAULTS[8]}) DEFAULT_USE_PLOTS_POSTPROC=$(eval echo ${DEFAULTS[9]}) +DEFAULT_USE_NETCDF=$(eval echo ${DEFAULTS[10]}) +DEFAULT_ENABLE_MMS=$(eval echo ${DEFAULTS[11]}) if [[ $DEFAULT_USE_MAKIE_POSTPROC -eq 0 ]]; then echo "Would you like to set up makie_post_processing? [y]/n" @@ -283,6 +285,82 @@ else fi fi + +if [[ $DEFAULT_USE_NETCDF -eq 0 ]]; then + echo "Would you like to enable optional NetCDF I/O (warning: using NetCDF sometimes" + echo "causes errors when using a local or system install of HDF5)? [y]/n" + read -p "> " input + echo + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + # $input must be empty, 'y' or 'n'. It is none of these, so ask for input + # again until we get a valid response. + echo + echo "$input is not a valid response: [y]/n" + read -p "> " input + echo + done + if [[ $input == "n" ]]; then + USE_NETCDF=1 + else + USE_NETCDF=0 + fi +else + echo "Would you like to enable optional NetCDF I/O (warning: using NetCDF sometimes" + echo "causes errors when using a local or system install of HDF5)? y/[n]" + read -p "> " input + echo + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + # $input must be empty, 'y' or 'n'. It is none of these, so ask for input + # again until we get a valid response. + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + echo + done + if [[ $input == "y" ]]; then + USE_NETCDF=0 + else + USE_NETCDF=1 + fi +fi + + +if [[ $DEFAULT_ENABLE_MMS -eq 0 ]]; then + echo "Would you like to enable MMS testing? [y]/n" + read -p "> " input + echo + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + # $input must be empty, 'y' or 'n'. It is none of these, so ask for input + # again until we get a valid response. + echo + echo "$input is not a valid response: [y]/n" + read -p "> " input + echo + done + if [[ $input == "n" ]]; then + ENABLE_MMS=1 + else + ENABLE_MMS=0 + fi +else + echo "Would you like to to enable MMS testing? y/[n]" + read -p "> " input + echo + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + # $input must be empty, 'y' or 'n'. It is none of these, so ask for input + # again until we get a valid response. + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + echo + done + if [[ $input == "y" ]]; then + ENABLE_MMS=0 + else + ENABLE_MMS=1 + fi +fi + if [[ $BATCH_SYSTEM -eq 0 ]]; then # Get the account to submit jobs with echo "Enter the account code used to submit jobs [$DEFAULT_ACCOUNT]:" @@ -419,7 +497,7 @@ echo # command, because passing as a prefix does not work (sometimes??) within a # bash script (even though as far as JTO knows it should work). export JULIA_DEPOT_PATH=$JULIA_DIRECTORY -$JULIA machines/shared/machine_setup.jl "$MACHINE" "$ACCOUNT" "$JULIA_DIRECTORY" "$DEFAULT_RUN_TIME" "$DEFAULT_NODES" "$DEFAULT_POSTPROC_TIME" "$DEFAULT_POSTPROC_MEMORY" "$DEFAULT_PARTITION" "$DEFAULT_QOS" "$USE_MAKIE_POSTPROC" "$USE_PLOTS_POSTPROC" +$JULIA machines/shared/machine_setup.jl "$MACHINE" "$ACCOUNT" "$JULIA_DIRECTORY" "$DEFAULT_RUN_TIME" "$DEFAULT_NODES" "$DEFAULT_POSTPROC_TIME" "$DEFAULT_POSTPROC_MEMORY" "$DEFAULT_PARTITION" "$DEFAULT_QOS" "$USE_MAKIE_POSTPROC" "$USE_PLOTS_POSTPROC" "$USE_NETCDF" "$ENABLE_MMS" if [ -f julia.env ]; then # Set up modules, JULIA_DEPOT_PATH, etc. to use for the rest of this script @@ -460,7 +538,26 @@ fi # We want to always add a couple of dependencies that are required to run the # tests in the top-level environment by just 'include()'ing the test scripts. # We also run the 'stage two' setup now if it is required. -SETUP_COMMAND="bin/julia --project -e 'import Pkg; Pkg.add([\"HDF5\", \"MPI\", \"MPIPreferences\", \"SpecialFunctions\"])" +SETUP_COMMAND="bin/julia --project -e 'import Pkg" +if [[ $USE_NETCDF -eq 1 || $ENABLE_MMS -eq 1 ]]; then + # Remove packages used by non-selected extensions in case they were installed previously + if [[ $USE_NETCDF -eq 1 ]]; then + SETUP_COMMAND="$SETUP_COMMAND; try Pkg.rm(\"NCDatasets\") catch end" + fi + if [[ $ENABLE_MMS -eq 1 ]]; then + SETUP_COMMAND="$SETUP_COMMAND; try Pkg.rm([\"Symbolics\", \"IfElse\"]) catch end" + fi +fi +SETUP_COMMAND="$SETUP_COMMAND; Pkg.add([\"HDF5\", \"MPI\", \"MPIPreferences\", \"SpecialFunctions\"" +if [[ $USE_NETCDF -eq 0 ]]; then + # Install NetCDF interface package NCDatasets to enable file_io_netcdf extension + SETUP_COMMAND="$SETUP_COMMAND, \"NCDatasets\"" +fi +if [[ $ENABLE_MMS -eq 0 ]]; then + # Install Symbolics and IfElse packages required by manufactured_solns_ext extension + SETUP_COMMAND="$SETUP_COMMAND, \"Symbolics\", \"IfElse\"" +fi +SETUP_COMMAND="$SETUP_COMMAND])" # [ -f ] tests if exists and is a file if [ -f machines/shared/machine_setup_stage_two.jl ]; then # A second setup stage exists, so run it. diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index 1859f17c4..f937e637f 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -17,7 +17,9 @@ default_settings["base"] = Dict("default_run_time"=>"24:00:00", "default_partition"=>"", "default_qos"=>"", "use_makie"=>"1", - "use_plots"=>"1") + "use_plots"=>"1", + "use_netcdf"=>"1", + "enable_mms"=>"1") # No batch system steup for "generic-pc" default_settings["generic-pc"] = merge(default_settings["base"], Dict("default_run_time"=>"0:00:00", @@ -45,6 +47,8 @@ default_settings["marconi"] = merge(default_settings["base"], default_qos::String; use_makie::String; use_plots::String; + use_netcdf::String; + enable_mms::String; no_force_exit::Bool=false, interactive::Bool=true) @@ -108,6 +112,10 @@ re-running this function (if you want to). The arguments are: means no). * `use_plots` indicates whether plots_post_processing has been enabled ("0" means yes, "1" means no). +* `use_netcdf` indicates whether NetCDF I/O has been enabled ("0" means yes, "1" means + no). +* `enable_mms` indicates whether MMS testing has been enabled ("0" means yes, "1" means + no). Currently supported machines: * `"generic-pc"` - A generic personal computer. Set up for interactive use, rather than @@ -131,7 +139,9 @@ function machine_setup_moment_kinetics(machine::String, default_partition::String, default_qos::String, use_makie::String, - use_plots::String; + use_plots::String, + use_netcdf::String, + enable_mms::String; no_force_exit::Bool=false, interactive::Bool=true) @@ -226,6 +236,8 @@ function machine_setup_moment_kinetics(machine::String, mk_preferences["account"] = account mk_preferences["use_makie"] = use_makie mk_preferences["use_plots"] = use_plots + mk_preferences["use_netcdf"] = use_netcdf + mk_preferences["enable_mms"] = enable_mms open(local_preferences_filename, "w") do io TOML.print(io, local_preferences, sorted=true) end @@ -382,7 +394,7 @@ if abspath(PROGRAM_FILE) == @__FILE__ for setting ∈ ("default_run_time", "default_nodes", "default_postproc_time", "default_postproc_memory", "default_partition", "default_partition", "account", "julia_directory", "use_makie", - "use_plots") + "use_plots", "use_netcdf", "enable_mms") d[setting] = get(existing_settings, setting, d[setting]) end @@ -390,7 +402,8 @@ if abspath(PROGRAM_FILE) == @__FILE__ d["default_postproc_time"], "\" \"", d["default_postproc_memory"], "\" \"", d["default_partition"], "\" \"", d["default_qos"], "\" \"", d["account"], "\" \"", d["julia_directory"], "\" \"", d["use_makie"], - "\" \"", d["use_plots"], "\"") + "\" \"", d["use_plots"], "\" \"", d["use_netcdf"], "\" \"", + d["enable_mms"], "\"") exit(0) end diff --git a/moment_kinetics/Project.toml b/moment_kinetics/Project.toml index fde1b1797..4bfffe581 100644 --- a/moment_kinetics/Project.toml +++ b/moment_kinetics/Project.toml @@ -39,12 +39,13 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" [weakdeps] +IfElse = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173" NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7" [extensions] file_io_netcdf = "NCDatasets" -manufactured_solns_ext = "Symbolics" +manufactured_solns_ext = ["Symbolics", "IfElse"] [compat] HDF5_jll = "<1.14, >=1.15" From 0ff26aef88c8e70152ff4df670235135ed413fbe Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 28 Dec 2023 20:33:11 +0000 Subject: [PATCH 26/80] Save response of whether to download HDF5 as the default --- machines/generic-pc/compile_dependencies.sh | 49 ++++++++++++++----- .../generic-pc/machine_setup_stage_two.jl | 25 +++++++++- machines/shared/get_mk_preference.jl | 22 +++++++++ machines/shared/machine_setup.jl | 4 +- machines/shared/set_mk_preference.jl | 19 +++++++ 5 files changed, 104 insertions(+), 15 deletions(-) create mode 100644 machines/shared/get_mk_preference.jl create mode 100644 machines/shared/set_mk_preference.jl diff --git a/machines/generic-pc/compile_dependencies.sh b/machines/generic-pc/compile_dependencies.sh index 2c2d107c6..e0c78b7a8 100755 --- a/machines/generic-pc/compile_dependencies.sh +++ b/machines/generic-pc/compile_dependencies.sh @@ -2,27 +2,54 @@ set -e +if [ -f julia.env ]; then + # Set up modules, JULIA_DEPOT_PATH, etc. to use for the rest of this script + source julia.env +fi cd machines/artifacts/ ARTIFACT_DIR=$PWD # HDF5 ###### -echo "Do you want to download, and compile a local version of HDF5 (if you do" -echo "not do this, you will be given the option to choose an HDF5 library to" -echo "link later)? [y]/n" -read -p "> " input -while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - echo - echo "$input is not a valid response: y/[n]" +# Get default response for whether to download/build HDF5 +DEFAULT_BUILDHDF5=$(../../bin/julia ../shared/get_mk_preference.jl build_hdf5 0) + +if [[ $DEFAULT_BUILDHDF5 -eq 0 ]]; then + echo "Do you want to download, and compile a local version of HDF5 (if you do" + echo "not do this, you will be given the option to choose an HDF5 library to" + echo "link later)? [y]/n" read -p "> " input -done -if [[ -z $input || $input == "y" ]]; then - BUILDHDF5=0 + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + done + if [[ -z $input || $input == "y" ]]; then + BUILDHDF5=0 + else + BUILDHDF5=1 + fi else - BUILDHDF5=1 + echo "Do you want to download, and compile a local version of HDF5 (if you do" + echo "not do this, you will be given the option to choose an HDF5 library to" + echo "link later)? y/[n]" + read -p "> " input + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + done + if [[ -z $input || $input == "n" ]]; then + BUILDHDF5=1 + else + BUILDHDF5=0 + fi fi +# Save current response for whether to download/build HDF5 as default +../../bin/julia ../shared/set_mk_preference.jl build_hdf5 $BUILDHDF5 + if [[ BUILDHDF5 -eq 1 && -d hdf5-build ]]; then rm -r hdf5-build elif [[ BUILDHDF5 -eq 0 && -d hdf5-build ]]; then diff --git a/machines/generic-pc/machine_setup_stage_two.jl b/machines/generic-pc/machine_setup_stage_two.jl index 936d098b9..2c668a432 100644 --- a/machines/generic-pc/machine_setup_stage_two.jl +++ b/machines/generic-pc/machine_setup_stage_two.jl @@ -50,7 +50,21 @@ if isdir(local_hdf5_install_dir) hdf5_lib_hl = joinpath(local_hdf5_install_dir, "libhdf5_hl.so") else println("\n** Setting up to use system HDF5\n") + default_hdf5_dir = get(ENV, "HDF5_DIR", "") # try to find a path to a system hdf5, may not work on all systems + + using TOML + repo_dir = dirname(dirname(dirname(@__FILE__))) + local_preferences_filename = joinpath(repo_dir, "LocalPreferences.toml") + if ispath(local_preferences_filename) + local_preferences = TOML.parsefile(local_preferences_filename) + else + local_preferences = Dict{String,Any}() + end + mk_preferences = get(local_preferences, "moment_kinetics", Dict{String,String}()) + println("mk_preferences ", mk_preferences) + default_hdf5_dir = get(mk_preferences, "hdf5_dir", default_hdf5_dir) + hdf5_dir = "" hdf5_lib = "" hdf5_lib_hl = "" @@ -90,9 +104,16 @@ else end end end + + mk_preferences["hdf5_dir"] = hdf5_dir + open(local_preferences_filename, "w") do io + TOML.print(io, local_preferences, sorted=true) + end end -if hdf5_dir != "default" - using HDF5 +using HDF5 +if hdf5_dir == "default" + HDF5.API.set_libraries!() +else HDF5.API.set_libraries!(hdf5_lib, hdf5_lib_hl) end diff --git a/machines/shared/get_mk_preference.jl b/machines/shared/get_mk_preference.jl new file mode 100644 index 000000000..ae0acdfa6 --- /dev/null +++ b/machines/shared/get_mk_preference.jl @@ -0,0 +1,22 @@ +# Get string-valued integer-valued preference for moment_kinetics from LocalPreferences.toml + +using TOML + +preference_name = ARGS[1] +if length(ARGS) > 1 + default = ARGS[2] +else + default = "1" +end + +top_level_directory = dirname(dirname(dirname(@__FILE__))) +local_preferences_filename = joinpath(top_level_directory, "LocalPreferences.toml") +if ispath(local_preferences_filename) + local_preferences = TOML.parsefile(local_preferences_filename) +else + local_preferences = Dict{String,Any}() +end +mk_section = get(local_preferences, "moment_kinetics", Dict{String,Any}()) +preference = get(mk_section, preference_name, default) + +println(preference) diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index f937e637f..4ea009425 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -224,8 +224,8 @@ function machine_setup_moment_kinetics(machine::String, else local_preferences = Dict{String,Any}() end - # Always overwrite any existing preferences, to get a fresh setup - mk_preferences = local_preferences["moment_kinetics"] = Dict{String,String}() + mk_preferences = get(local_preferences, "moment_kinetics", Dict{String,String}()) + local_preferences["moment_kinetics"] = mk_preferences mk_preferences["julia_directory"] = julia_directory mk_preferences["default_run_time"] = default_run_time mk_preferences["default_nodes"] = default_nodes diff --git a/machines/shared/set_mk_preference.jl b/machines/shared/set_mk_preference.jl new file mode 100644 index 000000000..a4a643442 --- /dev/null +++ b/machines/shared/set_mk_preference.jl @@ -0,0 +1,19 @@ +# Set a string-valued preference for moment_kinetics in LocalPreferences.toml + +using TOML + +preference_name = ARGS[1] +preference_value = ARGS[2] + +top_level_directory = dirname(dirname(dirname(@__FILE__))) +local_preferences_filename = joinpath(top_level_directory, "LocalPreferences.toml") +if ispath(local_preferences_filename) + local_preferences = TOML.parsefile(local_preferences_filename) +else + local_preferences = Dict{String,Any}() +end +mk_section = get(local_preferences, "moment_kinetics", Dict{String,Any}()) +mk_section[preference_name] = preference_value +open(local_preferences_filename, "w") do io + TOML.print(io, local_preferences, sorted=true) +end From 091d96fa4311d5c785200e54e1d3c7269614d8dc Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 28 Dec 2023 22:10:42 +0000 Subject: [PATCH 27/80] Move post processing packages to nested subdirectories Allows for setting up separate project directories for post-processing, which will be useful for example when the different projects need different optimization flags. --- .gitignore | 2 ++ machines/machine_setup.sh | 8 ++++---- .../{ => makie_post_processing}/Project.toml | 0 .../src/makie_post_processing.jl | 0 .../{ => makie_post_processing}/src/shared_utils.jl | 0 .../{ => plots_post_processing}/Project.toml | 0 .../{ => plots_post_processing}/src/plot_MMS_sequence.jl | 0 .../{ => plots_post_processing}/src/plot_sequence.jl | 0 .../src/plots_post_processing.jl | 0 .../src/post_processing_input.jl | 0 .../plots_post_processing/src/shared_utils.jl | 1 + plots_post_processing/post_processing_input.jl | 2 +- plots_post_processing/src/shared_utils.jl | 1 - 13 files changed, 8 insertions(+), 6 deletions(-) rename makie_post_processing/{ => makie_post_processing}/Project.toml (100%) rename makie_post_processing/{ => makie_post_processing}/src/makie_post_processing.jl (100%) rename makie_post_processing/{ => makie_post_processing}/src/shared_utils.jl (100%) rename plots_post_processing/{ => plots_post_processing}/Project.toml (100%) rename plots_post_processing/{ => plots_post_processing}/src/plot_MMS_sequence.jl (100%) rename plots_post_processing/{ => plots_post_processing}/src/plot_sequence.jl (100%) rename plots_post_processing/{ => plots_post_processing}/src/plots_post_processing.jl (100%) rename plots_post_processing/{ => plots_post_processing}/src/post_processing_input.jl (100%) create mode 120000 plots_post_processing/plots_post_processing/src/shared_utils.jl delete mode 120000 plots_post_processing/src/shared_utils.jl diff --git a/.gitignore b/.gitignore index 7d781422c..6165f7974 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ performance-tests/config.toml Manifest.toml LocalPreferences.toml /Project.toml +/makie_post_processing/Project.toml +/plots_post_processing/Project.toml /julia.env /.julia bin diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 1814822ea..14d054a03 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -575,9 +575,9 @@ bin/julia --project -e 'import Pkg; Pkg.develop(path="moment_kinetics"); Pkg.pre if [[ $USE_MAKIE_POSTPROC -eq 0 ]]; then echo "Setting up makie_post_processing" if [[ $BATCH_SYSTEM -eq 0 ]]; then - bin/julia --project=makie_post_processing/ -e 'import Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.precompile()' + bin/julia --project=makie_post_processing/ -e 'import Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.develop(path="makie_post_processing/makie_post_processing"); Pkg.precompile()' else - bin/julia --project -e 'import Pkg; Pkg.develop(path="makie_post_processing")' + bin/julia --project -e 'import Pkg; Pkg.develop(path="makie_post_processing/makie_post_processing")' fi else if [[ $BATCH_SYSTEM -eq 1 ]]; then @@ -588,9 +588,9 @@ fi if [[ $USE_PLOTS_POSTPROC -eq 0 ]]; then echo "Setting up plots_post_processing" if [[ $BATCH_SYSTEM -eq 0 ]]; then - bin/julia --project=plots_post_processing/ -e 'import Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.precompile()' + bin/julia --project=plots_post_processing/ -e 'import Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.develop(path="plots_post_processing/plots_post_processing"); Pkg.precompile()' else - bin/julia --project -e 'import Pkg; Pkg.develop(path="plots_post_processing")' + bin/julia --project -e 'import Pkg; Pkg.develop(path="plots_post_processing/plots_post_processing")' fi else if [[ $BATCH_SYSTEM -eq 1 ]]; then diff --git a/makie_post_processing/Project.toml b/makie_post_processing/makie_post_processing/Project.toml similarity index 100% rename from makie_post_processing/Project.toml rename to makie_post_processing/makie_post_processing/Project.toml diff --git a/makie_post_processing/src/makie_post_processing.jl b/makie_post_processing/makie_post_processing/src/makie_post_processing.jl similarity index 100% rename from makie_post_processing/src/makie_post_processing.jl rename to makie_post_processing/makie_post_processing/src/makie_post_processing.jl diff --git a/makie_post_processing/src/shared_utils.jl b/makie_post_processing/makie_post_processing/src/shared_utils.jl similarity index 100% rename from makie_post_processing/src/shared_utils.jl rename to makie_post_processing/makie_post_processing/src/shared_utils.jl diff --git a/plots_post_processing/Project.toml b/plots_post_processing/plots_post_processing/Project.toml similarity index 100% rename from plots_post_processing/Project.toml rename to plots_post_processing/plots_post_processing/Project.toml diff --git a/plots_post_processing/src/plot_MMS_sequence.jl b/plots_post_processing/plots_post_processing/src/plot_MMS_sequence.jl similarity index 100% rename from plots_post_processing/src/plot_MMS_sequence.jl rename to plots_post_processing/plots_post_processing/src/plot_MMS_sequence.jl diff --git a/plots_post_processing/src/plot_sequence.jl b/plots_post_processing/plots_post_processing/src/plot_sequence.jl similarity index 100% rename from plots_post_processing/src/plot_sequence.jl rename to plots_post_processing/plots_post_processing/src/plot_sequence.jl diff --git a/plots_post_processing/src/plots_post_processing.jl b/plots_post_processing/plots_post_processing/src/plots_post_processing.jl similarity index 100% rename from plots_post_processing/src/plots_post_processing.jl rename to plots_post_processing/plots_post_processing/src/plots_post_processing.jl diff --git a/plots_post_processing/src/post_processing_input.jl b/plots_post_processing/plots_post_processing/src/post_processing_input.jl similarity index 100% rename from plots_post_processing/src/post_processing_input.jl rename to plots_post_processing/plots_post_processing/src/post_processing_input.jl diff --git a/plots_post_processing/plots_post_processing/src/shared_utils.jl b/plots_post_processing/plots_post_processing/src/shared_utils.jl new file mode 120000 index 000000000..abdfe611b --- /dev/null +++ b/plots_post_processing/plots_post_processing/src/shared_utils.jl @@ -0,0 +1 @@ +../../../makie_post_processing/makie_post_processing/src/shared_utils.jl \ No newline at end of file diff --git a/plots_post_processing/post_processing_input.jl b/plots_post_processing/post_processing_input.jl index 77acb92c8..9ead0fd88 120000 --- a/plots_post_processing/post_processing_input.jl +++ b/plots_post_processing/post_processing_input.jl @@ -1 +1 @@ -src/post_processing_input.jl \ No newline at end of file +plots_post_processing/src/post_processing_input.jl \ No newline at end of file diff --git a/plots_post_processing/src/shared_utils.jl b/plots_post_processing/src/shared_utils.jl deleted file mode 120000 index 18ff3cde6..000000000 --- a/plots_post_processing/src/shared_utils.jl +++ /dev/null @@ -1 +0,0 @@ -../../makie_post_processing/src/shared_utils.jl \ No newline at end of file From 61524cbb65123d0e10edd5edcbf2ee37d425334b Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 28 Dec 2023 22:30:51 +0000 Subject: [PATCH 28/80] Set optimization flags for setup Ensures precompilation is done with the same flags as will be used later (or at least should be). This is important as precompilation of dependencies will be (or may be) re-done if the flags change. --- machines/machine_setup.sh | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 14d054a03..e8102d115 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -375,6 +375,19 @@ else ACCOUNT="" fi +if [[ $BATCH_SYSTEM -eq 0 ]]; then + # Batch systems can (conveniently) use different optimization flags for + # running simulations and for post-processing. + OPTIMIZATION_FLAGS="-O3 --check-bounds=no" + POSTPROC_OPTIMIZATION_FLAGS="-O3" +else + # On interactive systems which use the same project for running simulations + # and for post-processing, both should use the same optimization flags to + # avoid invalidating precompiled dependencies. + OPTIMIZATION_FLAGS="-O3" + POSTPROC_OPTIMIZATION_FLAGS=$OPTIMIZATION_FLAGS +fi + # Get the location for the .julia directory, in case this has to have a # non-default value, e.g. because the user's home directory is not accessible # from compute nodes. @@ -538,7 +551,7 @@ fi # We want to always add a couple of dependencies that are required to run the # tests in the top-level environment by just 'include()'ing the test scripts. # We also run the 'stage two' setup now if it is required. -SETUP_COMMAND="bin/julia --project -e 'import Pkg" +SETUP_COMMAND="bin/julia --project $OPTIMIZATION_FLAGS -e 'import Pkg" if [[ $USE_NETCDF -eq 1 || $ENABLE_MMS -eq 1 ]]; then # Remove packages used by non-selected extensions in case they were installed previously if [[ $USE_NETCDF -eq 1 ]]; then @@ -570,31 +583,31 @@ SETUP_COMMAND="$SETUP_COMMAND'" # Add the closing quote mark eval "$SETUP_COMMAND" # Add moment_kinetics package to the working project -bin/julia --project -e 'import Pkg; Pkg.develop(path="moment_kinetics"); Pkg.precompile()' +bin/julia --project $OPTIMIZATION_FLAGS -e 'import Pkg; Pkg.develop(path="moment_kinetics"); Pkg.precompile()' if [[ $USE_MAKIE_POSTPROC -eq 0 ]]; then echo "Setting up makie_post_processing" if [[ $BATCH_SYSTEM -eq 0 ]]; then - bin/julia --project=makie_post_processing/ -e 'import Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.develop(path="makie_post_processing/makie_post_processing"); Pkg.precompile()' + bin/julia --project=makie_post_processing/ $POSTPROC_OPTIMIZATION_FLAGS -e 'import Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.develop(path="makie_post_processing/makie_post_processing"); Pkg.precompile()' else - bin/julia --project -e 'import Pkg; Pkg.develop(path="makie_post_processing/makie_post_processing")' + bin/julia --project -e $OPTIMIZATION_FLAGS 'import Pkg; Pkg.develop(path="makie_post_processing/makie_post_processing")' fi else if [[ $BATCH_SYSTEM -eq 1 ]]; then - bin/julia --project -e 'import Pkg; try Pkg.rm("makie_post_processing") catch end' + bin/julia --project $OPTIMIZATION_FLAGS -e 'import Pkg; try Pkg.rm("makie_post_processing") catch end' fi fi if [[ $USE_PLOTS_POSTPROC -eq 0 ]]; then echo "Setting up plots_post_processing" if [[ $BATCH_SYSTEM -eq 0 ]]; then - bin/julia --project=plots_post_processing/ -e 'import Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.develop(path="plots_post_processing/plots_post_processing"); Pkg.precompile()' + bin/julia --project=plots_post_processing/ $POSTPROC_OPTIMIZATION_FLAGS -e 'import Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.develop(path="plots_post_processing/plots_post_processing"); Pkg.precompile()' else - bin/julia --project -e 'import Pkg; Pkg.develop(path="plots_post_processing/plots_post_processing")' + bin/julia --project $OPTIMIZATION_FLAGS -e 'import Pkg; Pkg.develop(path="plots_post_processing/plots_post_processing")' fi else if [[ $BATCH_SYSTEM -eq 1 ]]; then - bin/julia --project -e 'import Pkg; try Pkg.rm("plots_post_processing") catch end' + bin/julia --project $OPTIMIZATION_FLAGS -e 'import Pkg; try Pkg.rm("plots_post_processing") catch end' fi fi From 4f714b3190a7f2777292be62c470a9d990e4f711 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Fri, 29 Dec 2023 12:04:22 +0000 Subject: [PATCH 29/80] Consolidate all machine_setup_stage_two.jl into a single script This should be simpler than maintaining a separate script for each machine. Now machine-specific settings are saved in a machine_settings.toml file for each configuration. --- .gitignore | 1 - machines/archer/machine_settings.toml | 27 +++ machines/archer/machine_setup_stage_two.jl | 51 ----- machines/generic-pc/machine_settings.toml | 4 + .../generic-pc/machine_setup_stage_two.jl | 135 ------------- machines/machine_setup.sh | 11 +- machines/marconi/machine_settings.toml | 4 + machines/marconi/machine_setup_stage_two.jl | 39 ---- machines/shared/machine_setup.jl | 41 +--- machines/shared/machine_setup_stage_two.jl | 177 ++++++++++++++++++ 10 files changed, 222 insertions(+), 268 deletions(-) create mode 100644 machines/archer/machine_settings.toml delete mode 100644 machines/archer/machine_setup_stage_two.jl create mode 100644 machines/generic-pc/machine_settings.toml delete mode 100644 machines/generic-pc/machine_setup_stage_two.jl create mode 100644 machines/marconi/machine_settings.toml delete mode 100644 machines/marconi/machine_setup_stage_two.jl create mode 100644 machines/shared/machine_setup_stage_two.jl diff --git a/.gitignore b/.gitignore index 6165f7974..0802b789b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ LocalPreferences.toml /.julia bin machines/shared/compile_dependencies.sh -machines/shared/machine_setup_stage_two.jl machines/artifacts /precompile-makie-post-processing-submit.sh /precompile-plots-post-processing-submit.sh diff --git a/machines/archer/machine_settings.toml b/machines/archer/machine_settings.toml new file mode 100644 index 000000000..b62da094e --- /dev/null +++ b/machines/archer/machine_settings.toml @@ -0,0 +1,27 @@ +# MPI settings +############## + +# For GNU compilers. +# Working using the versions as of 19/6/2023, with the following modules loaded: +# 1) craype/2.7.19 5) cray-libsci/22.12.1.1 9) cray-python/3.9.13.1 +# 2) cray-dsmml/0.2.2 6) PrgEnv-cray/8.3.3 10) cray-hdf5-parallel/1.12.2.1 +# 3) libfabric/1.12.1.2.2.0.0 7) gcc/11.2.0 +# 4) craype-network-ofi 8) cray-mpich/8.1.23 +mpi_library_names = "libmpi_gnu_91" +mpiexec = "srun" + +# For Cray compilers. +# As of 19/6/2023 does not seem to work (jobscript-precompile hangs while running +# precompile.jl, with no terminal output). Had the following modules loaded: +# 1) cce/15.0.0 5) craype-network-ofi 9) cray-python/3.9.13.1 +# 2) craype/2.7.19 6) cray-mpich/8.1.23 10) cray-hdf5-parallel/1.12.2.1 +# 3) cray-dsmml/0.2.2 7) cray-libsci/22.12.1.1 +# 4) libfabric/1.12.1.2.2.0.0 8) PrgEnv-cray/8.3.3 +#mpi_library_names = "libmpi_cray" +#mpiexec = "srun" + + +# HDF5 settings +############### + +hdf5_library_setting = "system" diff --git a/machines/archer/machine_setup_stage_two.jl b/machines/archer/machine_setup_stage_two.jl deleted file mode 100644 index ea5049b57..000000000 --- a/machines/archer/machine_setup_stage_two.jl +++ /dev/null @@ -1,51 +0,0 @@ -using Pkg - -# Instantiate packages so we can use MPIPreferences below -######################################################### - -println("\n** Getting dependencies\n") -Pkg.instantiate() -Pkg.resolve() - - -# HDF5 setup -############ - -println("\n** Setting up to use system HDF5\n") -hdf5_dir = ENV["HDF5_DIR"] # system hdf5 -using HDF5 -HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), joinpath(hdf5_dir, "libhdf5_hl.so")) - - -# MPI setup -########### - -println("\n** Setting up to use system MPI\n") -using MPIPreferences - -# For GNU compilers. -# Working using the versions as of 19/6/2023, with the following modules loaded: -# 1) craype/2.7.19 5) cray-libsci/22.12.1.1 9) cray-python/3.9.13.1 -# 2) cray-dsmml/0.2.2 6) PrgEnv-cray/8.3.3 10) cray-hdf5-parallel/1.12.2.1 -# 3) libfabric/1.12.1.2.2.0.0 7) gcc/11.2.0 -# 4) craype-network-ofi 8) cray-mpich/8.1.23 -MPIPreferences.use_system_binary(library_names="libmpi_gnu_91", mpiexec="srun") - -# For Cray compilers. -# As of 19/6/2023 does not seem to work (jobscript-precompile hangs while running -# precompile.jl, with no terminal output). Had the following modules loaded: -# 1) cce/15.0.0 5) craype-network-ofi 9) cray-python/3.9.13.1 -# 2) craype/2.7.19 6) cray-mpich/8.1.23 10) cray-hdf5-parallel/1.12.2.1 -# 3) cray-dsmml/0.2.2 7) cray-libsci/22.12.1.1 -# 4) libfabric/1.12.1.2.2.0.0 8) PrgEnv-cray/8.3.3 -#MPIPreferences.use_system_binary(library_names="libmpi_cray", mpiexec="srun") - - -# Force exit so Julia must be restarted -####################################### - -println() -println("************************************************************") -println("Julia must be restarted to use the updated MPI, exiting now.") -println("************************************************************") -exit(0) diff --git a/machines/generic-pc/machine_settings.toml b/machines/generic-pc/machine_settings.toml new file mode 100644 index 000000000..e0dce19dc --- /dev/null +++ b/machines/generic-pc/machine_settings.toml @@ -0,0 +1,4 @@ +# HDF5 settings +############### + +hdf5_library_setting = "prompt" diff --git a/machines/generic-pc/machine_setup_stage_two.jl b/machines/generic-pc/machine_setup_stage_two.jl deleted file mode 100644 index 2c668a432..000000000 --- a/machines/generic-pc/machine_setup_stage_two.jl +++ /dev/null @@ -1,135 +0,0 @@ -using Pkg - -""" - get_input_with_path_completion(message=nothing) - -Print `message` and get user input using the `read` utility from bash to allow -path-completion. - -Solution adapted from -https://discourse.julialang.org/t/collecting-all-output-from-shell-commands/15592/2 -""" -function get_input_with_path_completion(message=nothing) - if message !== nothing - println(message) - end - - # The `out` object is used by `pipeline()` to capture output from the shell command. - out = Pipe() - - # Use Julia's shell command functionality to actually run bash - not sure how to use - # `read` and get output from it without using bash. - run(pipeline(`bash -c "read -e -p '> ' USERINPUT; echo \$USERINPUT"`, stdout=out)) - - # Need to close `out.in` to be able to read from `out`. - close(out.in) - - # chomp removes the trailing '\n'. 'String()' converts Vector{Char} to a String. - input = chomp(String(read(out))) - - return input -end - -# Instantiate packages so we can use MPIPreferences below -######################################################### - -println("\n** Getting dependencies\n") -Pkg.instantiate() -Pkg.resolve() - - -# HDF5 setup -############ - -local_hdf5_install_dir = joinpath("machines", "artifacts", "hdf5-build", "lib") -if isdir(local_hdf5_install_dir) - local_hdf5_install_dir = realpath(local_hdf5_install_dir) - # We have downloaded and compiled HDF5, so link that - hdf5_dir = local_hdf5_install_dir - hdf5_lib = joinpath(local_hdf5_install_dir, "libhdf5.so") - hdf5_lib_hl = joinpath(local_hdf5_install_dir, "libhdf5_hl.so") -else - println("\n** Setting up to use system HDF5\n") - - default_hdf5_dir = get(ENV, "HDF5_DIR", "") # try to find a path to a system hdf5, may not work on all systems - - using TOML - repo_dir = dirname(dirname(dirname(@__FILE__))) - local_preferences_filename = joinpath(repo_dir, "LocalPreferences.toml") - if ispath(local_preferences_filename) - local_preferences = TOML.parsefile(local_preferences_filename) - else - local_preferences = Dict{String,Any}() - end - mk_preferences = get(local_preferences, "moment_kinetics", Dict{String,String}()) - println("mk_preferences ", mk_preferences) - default_hdf5_dir = get(mk_preferences, "hdf5_dir", default_hdf5_dir) - - hdf5_dir = "" - hdf5_lib = "" - hdf5_lib_hl = "" - while true - global hdf5_dir, hdf5_lib, hdf5_lib_hl - hdf5_dir = get_input_with_path_completion( - "\nAn HDF5 installation compiled with your system MPI is required to use\n" - * "parallel I/O. Enter the directory where the libhdf5.so and libhdf5_hl.so are\n" - * "located (enter 'default' to use the Julia-provided HDF5, which does not\n" - * "support parallel I/O): [$default_hdf5_dir]") - - if hdf5_dir == "" - hdf5_dir = default_hdf5_dir - end - - if hdf5_dir == "default" - break - end - - if isdir(hdf5_dir) - hdf5_dir = realpath(hdf5_dir) - end - hdf5_lib = joinpath(hdf5_dir, "libhdf5.so") - hdf5_lib_hl = joinpath(hdf5_dir, "libhdf5_hl.so") - if isfile(hdf5_lib) && isfile(hdf5_lib_hl) - break - else - # Remove trailing slash if it exists so that we can print a single trailing slash - # consistently - hdf5_dir = rstrip(hdf5_dir, '/') - print("HDF5 libraries not found in '$hdf5_dir/'.") - if !isfile(hdf5_lib) - print(" $hdf5_lib does not exist.") - end - if !isfile(hdf5_lib_hl) - print(" $hdf5_lib_hl does not exist.") - end - end - end - - mk_preferences["hdf5_dir"] = hdf5_dir - open(local_preferences_filename, "w") do io - TOML.print(io, local_preferences, sorted=true) - end -end -using HDF5 -if hdf5_dir == "default" - HDF5.API.set_libraries!() -else - HDF5.API.set_libraries!(hdf5_lib, hdf5_lib_hl) -end - -# MPI setup -########### - -println("\n** Setting up to use system MPI\n") -using MPIPreferences -MPIPreferences.use_system_binary() - - -# Force exit so Julia must be restarted -####################################### - -println() -println("************************************************************") -println("Julia must be restarted to use the updated MPI, exiting now.") -println("************************************************************") -exit(0) diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index e8102d115..f4f432fe8 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -570,16 +570,7 @@ if [[ $ENABLE_MMS -eq 0 ]]; then # Install Symbolics and IfElse packages required by manufactured_solns_ext extension SETUP_COMMAND="$SETUP_COMMAND, \"Symbolics\", \"IfElse\"" fi -SETUP_COMMAND="$SETUP_COMMAND])" -# [ -f ] tests if exists and is a file -if [ -f machines/shared/machine_setup_stage_two.jl ]; then - # A second setup stage exists, so run it. - # This does setup for HDF5 and MPI, possibly other things if necessary. - echo - echo "Including stage two setup" - SETUP_COMMAND="$SETUP_COMMAND; include(\"machines/shared/machine_setup_stage_two.jl\")" -fi -SETUP_COMMAND="$SETUP_COMMAND'" # Add the closing quote mark +SETUP_COMMAND="$SETUP_COMMAND]); include(\"machines/shared/machine_setup_stage_two.jl\")'" eval "$SETUP_COMMAND" # Add moment_kinetics package to the working project diff --git a/machines/marconi/machine_settings.toml b/machines/marconi/machine_settings.toml new file mode 100644 index 000000000..2224be1ab --- /dev/null +++ b/machines/marconi/machine_settings.toml @@ -0,0 +1,4 @@ +# HDF5 settings +############### + +hdf5_library_setting = "download" diff --git a/machines/marconi/machine_setup_stage_two.jl b/machines/marconi/machine_setup_stage_two.jl deleted file mode 100644 index e9b550a21..000000000 --- a/machines/marconi/machine_setup_stage_two.jl +++ /dev/null @@ -1,39 +0,0 @@ -# Instantiate packages so we can use MPIPreferences below -######################################################### - -using Pkg - -println("\n** Getting dependencies\n") -Pkg.instantiate() -Pkg.resolve() - - -repo_dir = dirname(dirname(dirname(@__FILE__))) -artifact_dir = joinpath(repo_dir, "machines", "artifacts") - -# HDF5 setup -############ - -println("\n** Setting up to use custom compiled HDF5\n") -hdf5_dir = joinpath(artifact_dir, "hdf5-build/") -using HDF5 -HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), joinpath(hdf5_dir, "libhdf5_hl.so")) - - -# MPI setup -########### - -println("\n** Setting up to use system MPI\n") -using MPIPreferences - -MPIPreferences.use_system_binary() - - -# Force exit so Julia must be restarted -####################################### - -println() -println("************************************************************") -println("Julia must be restarted to use the updated MPI, exiting now.") -println("************************************************************") -exit(0) diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index 4ea009425..c61126e32 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -226,6 +226,7 @@ function machine_setup_moment_kinetics(machine::String, end mk_preferences = get(local_preferences, "moment_kinetics", Dict{String,String}()) local_preferences["moment_kinetics"] = mk_preferences + mk_preferences["machine"] = machine mk_preferences["julia_directory"] = julia_directory mk_preferences["default_run_time"] = default_run_time mk_preferences["default_nodes"] = default_nodes @@ -250,13 +251,6 @@ function machine_setup_moment_kinetics(machine::String, compile_dependencies_relative_path) needs_compile_dependencies = false - # A second stage of setup may be needed after restarting Julia on some machines. - # If it is, set `needs_second_stage = true` in the machine-specific case below. - second_stage_relative_path = joinpath("machines", "shared", - "machine_setup_stage_two.jl") - second_stage_path = joinpath(repo_dir, second_stage_relative_path) - needs_second_stage = false - # Set this flag to true in the machine-specific branch below to require a # non-empty `account` setting needs_account = false @@ -265,8 +259,6 @@ function machine_setup_moment_kinetics(machine::String, # For generic-pc, run compile_dependencies.sh script to optionally download and # compile HDF5 needs_compile_dependencies = true - - needs_second_stage = true elseif machine == "archer" needs_account = true if julia_directory == "" @@ -275,19 +267,11 @@ function machine_setup_moment_kinetics(machine::String, * "directory and the `/home/` filesystem is not available on the " * "compute nodes.") end - - # Need to set JULIA_DEPOT_PATH so the `.julia` directory is on the /work - # filesystem (where it can be used on compute nodes, unlike /home) before - # setting up MPI and HDF5, so a second stage is required for archer. - needs_second_stage = true elseif machine == "marconi" needs_account = true # For marconi, need to run a script to compile HDF5 needs_compile_dependencies = true - - # Second stage is required for marconi to set up HDF5 and MPI - needs_second_stage = true else error("Unsupported machine '$machine'") end @@ -312,21 +296,14 @@ function machine_setup_moment_kinetics(machine::String, end end - if needs_second_stage - # Remove link if it exists already - islink(second_stage_path) && rm(second_stage_path) - - symlink(joinpath("..", machine, "machine_setup_stage_two.jl"), second_stage_path) - - if interactive - println() - println("***********************************************************************") - println("To complete setup, first `source julia.env` if it exists (so that") - println("`JULIA_DEPOT_PATH` is set correctly), then start Julia again (you can") - println("now use `bin/julia --project`) and to complete the setup run:") - println(" julia> include(\"$second_stage_relative_path\")") - println("***********************************************************************") - end + if interactive + println() + println("***********************************************************************") + println("To complete setup, first `source julia.env` if it exists (so that") + println("`JULIA_DEPOT_PATH` is set correctly), then start Julia again (you can") + println("now use `bin/julia --project`) and to complete the setup run:") + println(" julia> include(\"$second_stage_relative_path\")") + println("***********************************************************************") end if !no_force_exit diff --git a/machines/shared/machine_setup_stage_two.jl b/machines/shared/machine_setup_stage_two.jl new file mode 100644 index 000000000..8c030c90a --- /dev/null +++ b/machines/shared/machine_setup_stage_two.jl @@ -0,0 +1,177 @@ +using Pkg, TOML + + +""" + get_input_with_path_completion(message=nothing) + +Print `message` and get user input using the `read` utility from bash to allow +path-completion. + +Solution adapted from +https://discourse.julialang.org/t/collecting-all-output-from-shell-commands/15592/2 +""" +function get_input_with_path_completion(message=nothing) + if message !== nothing + println(message) + end + + # The `out` object is used by `pipeline()` to capture output from the shell command. + out = Pipe() + + # Use Julia's shell command functionality to actually run bash - not sure how to use + # `read` and get output from it without using bash. + run(pipeline(`bash -c "read -e -p '> ' USERINPUT; echo \$USERINPUT"`, stdout=out)) + + # Need to close `out.in` to be able to read from `out`. + close(out.in) + + # chomp removes the trailing '\n'. 'String()' converts Vector{Char} to a String. + input = chomp(String(read(out))) + + return input +end + + +repo_dir = dirname(dirname(dirname(@__FILE__))) +local_settings = TOML.parsefile(joinpath(repo_dir, "LocalPreferences.toml"))["moment_kinetics"] +machine = local_settings["machine"] +machine_dir = joinpath(repo_dir, "machines", machine) +machine_settings_filename = joinpath(machine_dir, "machine_settings.toml") +if isfile(machine_settings_filename) + machine_settings = TOML.parsefile(machine_settings_filename) +else + machine_settings = Dict{String,Any}() +end + + +# Instantiate packages so we can use MPIPreferences below +######################################################### + +println("\n** Getting dependencies\n") +Pkg.instantiate() +Pkg.resolve() + + +# MPI setup +########### + +println("\n** Setting up to use system MPI\n") +using MPIPreferences + +if "mpi_library_names" ∈ keys(machine_settings) || "mpiexec" ∈ keys(machine_settings) + MPIPreferences.use_system_binary(library_names=machine_settings["mpi_library_names"], + mpiexec=machine_settings["mpiexec"]) +else + # If settings for MPI library are not given explicitly, then auto-detection by + # MPIPreferences.use_system_binary() should work. + MPIPreferences.use_system_binary() +end + + +# HDF5 setup +############ + +println("\n** Setting up to use system HDF5\n") + +if machine_settings["hdf5_library_setting"] == "system" + hdf5_dir = ENV["HDF5_DIR"] # system hdf5 + using HDF5 + HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), + joinpath(hdf5_dir, "libhdf5_hl.so")) +elseif machine_settings["hdf5_library_setting"] == "download" + artifact_dir = joinpath(repo_dir, "machines", "artifacts") + hdf5_dir = joinpath(artifact_dir, "hdf5-build/") + using HDF5 + HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), + joinpath(hdf5_dir, "libhdf5_hl.so")) +elseif machine_settings["hdf5_library_setting"] == "prompt" + # Prompt user to select what HDF5 to use + local_hdf5_install_dir = joinpath("machines", "artifacts", "hdf5-build", "lib") + if isdir(local_hdf5_install_dir) + local_hdf5_install_dir = realpath(local_hdf5_install_dir) + # We have downloaded and compiled HDF5, so link that + hdf5_dir = local_hdf5_install_dir + hdf5_lib = joinpath(local_hdf5_install_dir, "libhdf5.so") + hdf5_lib_hl = joinpath(local_hdf5_install_dir, "libhdf5_hl.so") + else + println("\n** Setting up to use system HDF5\n") + + default_hdf5_dir = get(ENV, "HDF5_DIR", "") # try to find a path to a system hdf5, may not work on all systems + + using TOML + repo_dir = dirname(dirname(dirname(@__FILE__))) + local_preferences_filename = joinpath(repo_dir, "LocalPreferences.toml") + if ispath(local_preferences_filename) + local_preferences = TOML.parsefile(local_preferences_filename) + else + local_preferences = Dict{String,Any}() + end + mk_preferences = get(local_preferences, "moment_kinetics", Dict{String,String}()) + println("mk_preferences ", mk_preferences) + default_hdf5_dir = get(mk_preferences, "hdf5_dir", default_hdf5_dir) + + hdf5_dir = "" + hdf5_lib = "" + hdf5_lib_hl = "" + while true + global hdf5_dir, hdf5_lib, hdf5_lib_hl + hdf5_dir = get_input_with_path_completion( + "\nAn HDF5 installation compiled with your system MPI is required to use\n" + * "parallel I/O. Enter the directory where the libhdf5.so and libhdf5_hl.so are\n" + * "located (enter 'default' to use the Julia-provided HDF5, which does not\n" + * "support parallel I/O): [$default_hdf5_dir]") + + if hdf5_dir == "" + hdf5_dir = default_hdf5_dir + end + + if hdf5_dir == "default" + break + end + + if isdir(hdf5_dir) + hdf5_dir = realpath(hdf5_dir) + end + hdf5_lib = joinpath(hdf5_dir, "libhdf5.so") + hdf5_lib_hl = joinpath(hdf5_dir, "libhdf5_hl.so") + if isfile(hdf5_lib) && isfile(hdf5_lib_hl) + break + else + # Remove trailing slash if it exists so that we can print a single trailing slash + # consistently + hdf5_dir = rstrip(hdf5_dir, '/') + print("HDF5 libraries not found in '$hdf5_dir/'.") + if !isfile(hdf5_lib) + print(" $hdf5_lib does not exist.") + end + if !isfile(hdf5_lib_hl) + print(" $hdf5_lib_hl does not exist.") + end + end + end + + mk_preferences["hdf5_dir"] = hdf5_dir + open(local_preferences_filename, "w") do io + TOML.print(io, local_preferences, sorted=true) + end + end + using HDF5 + if hdf5_dir == "default" + HDF5.API.set_libraries!() + else + HDF5.API.set_libraries!(hdf5_lib, hdf5_lib_hl) + end +else + error("Unrecognized setting " + * "hdf5_library_setting=$(machine_settings["hdf5_library_setting"])") +end + + +# Force exit so Julia must be restarted +####################################### + +println() +println("************************************************************") +println("Julia must be restarted to use the updated MPI, exiting now.") +println("************************************************************") +exit(0) From e6058ac535501cbe5102e71ce2d8ecca87774e6b Mon Sep 17 00:00:00 2001 From: John Omotani Date: Fri, 29 Dec 2023 12:49:11 +0000 Subject: [PATCH 30/80] Save location of julia executable, use as default when rerunning setup Also simplifies the finding of the julia executable. The positional argument to machine_setup.sh and the '-d' flag are removed - the script just prompts the user for input, and remembers the settings so that they are provided as the default if the machine_setup.sh script is re-run. --- .gitignore | 1 + machines/machine_setup.sh | 187 ++++++++++++++++++++------------------ 2 files changed, 101 insertions(+), 87 deletions(-) diff --git a/.gitignore b/.gitignore index 0802b789b..d09eeea92 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ machines/artifacts /precompile-submit.sh /submit-restart.sh /submit-run.sh +/.julia_default.txt /.this_machine_name.txt moment_kinetics.so precompile-temp diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index f4f432fe8..bb7c6403e 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -5,22 +5,12 @@ set -e # Parse command line options # [See e.g. https://www.stackchief.com/tutorials/Bash%20Tutorial%3A%20getopts # for examples of how to use Bash's getopts] -FORCE_DOWNLOAD=1 -while getopts "hd" opt; do +while getopts "h" opt; do case $opt in h) echo "Setup moment_kinetics to run on a known machine" - echo "Usage: machine_setup.sh [-d] [/path/to/julia]" - echo - echo "If the path to julia is not given, you will be prompted for it." - echo " -d Force Julia to be downloaded even if julia executable is" - echo " present in \$PATH. Ignored if is passed" - echo " explicitly." exit 1 ;; - d) - FORCE_DOWNLOAD=0 - ;; esac done @@ -28,9 +18,17 @@ done # project environment that we can install packages to. touch Project.toml -# Get the positional argument as JULIA -# [See https://stackoverflow.com/a/13400237] -JULIA=${@:$OPTIND:1} +# If `julia` executable has been set up before, then that will be used as the +# default selection (unless -d flag is passed explicitly to download `julia`). +if [[ -f .julia_default.txt ]]; then + JULIA=$(cat .julia_default.txt) + if [[ ! -f $JULIA ]]; then + # Previously-used path to `julia` no longer exists, so ignore it and remove + # the stored path. + JULIA="" + echo > .julia_default.txt + fi +fi if [[ -f .this_machine_name.txt ]]; then DEFAULT_MACHINE=$(cat .this_machine_name.txt) @@ -53,83 +51,98 @@ mkdir -p machines/artifacts # Make sure $JULIA is set # Note [ -z "$VAR" ] tests if $VAR is empty. Need the quotes to ensure that the # contents of $VAR are not evaluated if it is not empty. -if [ -z "$JULIA" ]; then +if [ -z "$JULIA" ]; then # Try to find a sensible default if there is one in the $PATH if [ $(command -v julia) ]; then JULIA=$(command -v julia) fi - # REQUEST_INPUT will be set to 1 if the path to Julia is not needed because - # Julia was downloaded. If it stays at zero, request a path to Julia after - # looking for a good default value in this block. - REQUEST_INPUT=0 - if [[ -z "$JULIA" || $FORCE_DOWNLOAD -eq 0 ]]; then - if [[ $FORCE_DOWNLOAD -eq 0 ]]; then - # '-d' flag was passed, so carry on and download Julia without prompting - input="y" - else - echo "No 'julia' found in \$PATH. Would you like to download Julia? [y]/n" - read -p "> " input - echo - fi - while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - # $input must be empty, 'y' or 'n'. It is none of these, so ask for input - # again until we get a valid response. - echo - echo "$input is not a valid response: [y]/n" - read -p "> " input - echo - done - - if [[ -z $input || $input == "y" ]]; then - # Need machine name to get the right get-julia.sh script. - # Not ideal to get input here as in the 'proper' place below we can check - # for a valid input for $MACHINE and list the allowed values (although - # there will be an error here if the input is not valid as - # 'machines/$MACHINE/get-julia.sh would not exist). However the checking - # requires Julia, so cannot do it here as we need the value in order to - # download Julia (need to know the machine so that we download the right - # set of binaries for the OS, architecture, etc. - while [ -z $MACHINE ]; do - echo "Enter name of the machine to set up [$DEFAULT_MACHINE]:" - read -p "> " MACHINE - echo - if [ -z $MACHINE ]; then - MACHINE=$DEFAULT_MACHINE - fi - done - - # Download a version of Julia that is correct for this machine. - # Here the user can specify which version of Julia they want to use in - # case the latest stable versionis not wanted. - echo "Enter the version of Julia to download [latest]:" - read -p "> " version - - # The get-julia.sh script for $MACHINE should download and extract Julia - # (into machines/artifacts/). It prints the path to the 'julia' - # executable, which we store in $JULIA. - JULIA=$(machines/$MACHINE/get-julia.sh $version) - - # Make sure $JULIA is actually an executable - # Note 'command -v foo' returns success if foo is an executable and failure - # otherwise, except that if no argument is passed 'command -v' returns success, - # so we also need to check that $JULIA is not empty. - if [[ -z "$JULIA" || ! $(command -v $JULIA) ]]; then - echo "Failed to download Julia. '$JULIA' is not an executable" - exit 1 - fi - REQUEST_INPUT=1 - fi +fi +# REQUEST_INPUT will be set to 1 if the path to Julia is not needed because +# Julia was downloaded. If it stays at zero, request a path to Julia. +REQUEST_INPUT=0 +DOWNLOAD_JULIA=1 +if [[ -z "$JULIA" ]]; then + echo "No previously-used 'julia' or 'julia' found in \$PATH. Would you like" + echo "to download Julia? [y]/n" + read -p "> " input + echo + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + # $input must be empty, 'y' or 'n'. It is none of these, so ask for input + # again until we get a valid response. + echo + echo "$input is not a valid response: [y]/n" + read -p "> " input + echo + done + if [[ -z $input || $input == "y" ]]; then + DOWNLOAD_JULIA=0 + fi +else + echo "Existing 'julia' is available ($JULIA). Would you like to download" + echo "Julia anyway? y/[n]" + read -p "> " input + echo + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + # $input must be empty, 'y' or 'n'. It is none of these, so ask for input + # again until we get a valid response. + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + echo + done + if [[ $input == "y" ]]; then + DOWNLOAD_JULIA=0 fi - if [ $REQUEST_INPUT -eq 0 ]; then - # Have not downloaded Julia, so need to get the path to a 'julia' - # executable from the user. If 'julia' was found in $PATH, its location is - # stored in $JULIA at the moment, so use this as the default value. - echo "Please enter the path to the Julia executable to be used ['$JULIA']:" - # Use '-e' option to get path auto-completion - read -e -p "> " input - if [ ! -z "$input" ]; then - JULIA=$input +fi +if [[ $DOWNLOAD_JULIA -eq 0 ]]; then + # Need machine name to get the right get-julia.sh script. + # Not ideal to get input here as in the 'proper' place below we can check + # for a valid input for $MACHINE and list the allowed values (although + # there will be an error here if the input is not valid as + # 'machines/$MACHINE/get-julia.sh would not exist). However the checking + # requires Julia, so cannot do it here as we need the value in order to + # download Julia (need to know the machine so that we download the right + # set of binaries for the OS, architecture, etc. + while [ -z $MACHINE ]; do + echo "Enter name of the machine to set up [$DEFAULT_MACHINE]:" + read -p "> " MACHINE + echo + if [ -z $MACHINE ]; then + MACHINE=$DEFAULT_MACHINE fi + done + + # Download a version of Julia that is correct for this machine. + # Here the user can specify which version of Julia they want to use in + # case the latest stable versionis not wanted. + echo "Enter the version of Julia to download [latest]:" + read -p "> " version + + # The get-julia.sh script for $MACHINE should download and extract Julia + # (into machines/artifacts/). It prints the path to the 'julia' + # executable, which we store in $JULIA. + JULIA=$(machines/$MACHINE/get-julia.sh $version) + + # Make sure $JULIA is actually an executable + # Note 'command -v foo' returns success if foo is an executable and failure + # otherwise, except that if no argument is passed 'command -v' returns success, + # so we also need to check that $JULIA is not empty. + if [[ -z "$JULIA" || ! $(command -v $JULIA) ]]; then + echo "Failed to download Julia. '$JULIA' is not an executable" + exit 1 + fi + REQUEST_INPUT=1 +fi +if [ $REQUEST_INPUT -eq 0 ]; then + # Have not downloaded Julia, so need to get the path to a 'julia' + # executable from the user. If 'julia' was specified before, or found in + # $PATH, its location is stored in $JULIA at the moment, so use this as the + # default value. + echo "Please enter the path to the Julia executable to be used ['$JULIA']:" + # Use '-e' option to get path auto-completion + read -e -p "> " input + if [ ! -z "$input" ]; then + JULIA=$input fi fi # Ensure we have the resolved path for $JULIA, to avoid potentially creating a @@ -150,13 +163,13 @@ while [[ -z "$JULIA" || ! $(command -v $JULIA) ]]; do read -e -p "> " JULIA done # Convert $JULIA (which might be a relative path) to an absolute path. -# '-m' to avoid erroring if the directory does not exist already. # '-s' to skip resolving symlinks (if we did resolve symlinks, it might make # the path look different than expected). -JULIA=$(realpath -m -s $JULIA) +JULIA=$(realpath -s $JULIA) echo echo "Using Julia at $JULIA" echo +echo "$JULIA" > .julia_default.txt # If Julia was downloaded by this script, $MACHINE is already set. Otherwise, # need to get and check its value here. From 7e6c887ced23181fc6a14a062031dd27871fbfe2 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Fri, 29 Dec 2023 13:34:54 +0000 Subject: [PATCH 31/80] Fix download of julia for generic-pc, support downloading for macOS --- .../jobscript-postprocess-plotsjl.template | 2 +- .../archer/jobscript-postprocess.template | 2 +- ...-precompile-makie-post-processing.template | 2 +- .../jobscript-precompile-no-run.template | 2 +- ...-precompile-plots-post-processing.template | 2 +- machines/archer/jobscript-precompile.template | 2 +- machines/archer/jobscript-restart.template | 2 +- machines/archer/jobscript-run.template | 2 +- machines/generic-pc/compile_dependencies.sh | 2 +- machines/generic-pc/get-julia.sh | 30 ++++++++++++++++ machines/machine_setup.sh | 2 +- machines/marconi/compile_dependencies.sh | 2 +- machines/marconi/get-julia.sh | 2 +- .../jobscript-postprocess-plotsjl.template | 2 +- .../marconi/jobscript-postprocess.template | 2 +- ...-precompile-makie-post-processing.template | 2 +- .../jobscript-precompile-no-run.template | 2 +- ...-precompile-plots-post-processing.template | 2 +- .../marconi/jobscript-precompile.template | 2 +- machines/marconi/jobscript-restart.template | 2 +- machines/marconi/jobscript-run.template | 2 +- machines/shared/get-julia-linux-x86_64.sh | 2 +- machines/shared/get-julia-macos.sh | 35 +++++++++++++++++++ machines/shared/get-julia.py | 4 +++ machines/shared/machine_setup.jl | 2 +- ...precompile-makie-post-processing-submit.sh | 2 +- ...precompile-plots-post-processing-submit.sh | 2 +- machines/shared/precompile-submit.sh | 2 +- machines/shared/submit-restart.sh | 2 +- machines/shared/submit-run.sh | 2 +- 30 files changed, 96 insertions(+), 27 deletions(-) create mode 100755 machines/generic-pc/get-julia.sh create mode 100755 machines/shared/get-julia-macos.sh diff --git a/machines/archer/jobscript-postprocess-plotsjl.template b/machines/archer/jobscript-postprocess-plotsjl.template index 4f43a2f77..14da865f2 100644 --- a/machines/archer/jobscript-postprocess-plotsjl.template +++ b/machines/archer/jobscript-postprocess-plotsjl.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --mem=POSTPROCMEMORY #SBATCH --time=POSTPROCTIME diff --git a/machines/archer/jobscript-postprocess.template b/machines/archer/jobscript-postprocess.template index f63441bff..8fde6dc7b 100644 --- a/machines/archer/jobscript-postprocess.template +++ b/machines/archer/jobscript-postprocess.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --mem=POSTPROCMEMORY #SBATCH --time=POSTPROCTIME diff --git a/machines/archer/jobscript-precompile-makie-post-processing.template b/machines/archer/jobscript-precompile-makie-post-processing.template index 44f70f5f1..480e138f2 100644 --- a/machines/archer/jobscript-precompile-makie-post-processing.template +++ b/machines/archer/jobscript-precompile-makie-post-processing.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --mem=64G #SBATCH --ntasks=1 diff --git a/machines/archer/jobscript-precompile-no-run.template b/machines/archer/jobscript-precompile-no-run.template index 8e19d328c..bb444987b 100644 --- a/machines/archer/jobscript-precompile-no-run.template +++ b/machines/archer/jobscript-precompile-no-run.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --mem=64G #SBATCH --ntasks=1 diff --git a/machines/archer/jobscript-precompile-plots-post-processing.template b/machines/archer/jobscript-precompile-plots-post-processing.template index 6d7ae4995..fdd54c4fc 100644 --- a/machines/archer/jobscript-precompile-plots-post-processing.template +++ b/machines/archer/jobscript-precompile-plots-post-processing.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --mem=64G #SBATCH --ntasks=1 diff --git a/machines/archer/jobscript-precompile.template b/machines/archer/jobscript-precompile.template index 229518d5c..e4d681ed6 100644 --- a/machines/archer/jobscript-precompile.template +++ b/machines/archer/jobscript-precompile.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --mem=64G #SBATCH --ntasks=1 diff --git a/machines/archer/jobscript-restart.template b/machines/archer/jobscript-restart.template index 52501d169..6ac1c50f9 100644 --- a/machines/archer/jobscript-restart.template +++ b/machines/archer/jobscript-restart.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --nodes=NODES #SBATCH --cpus-per-task=1 diff --git a/machines/archer/jobscript-run.template b/machines/archer/jobscript-run.template index f88dbdfa6..ea2787bf0 100644 --- a/machines/archer/jobscript-run.template +++ b/machines/archer/jobscript-run.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --nodes=NODES #SBATCH --cpus-per-task=1 diff --git a/machines/generic-pc/compile_dependencies.sh b/machines/generic-pc/compile_dependencies.sh index e0c78b7a8..1159083b6 100755 --- a/machines/generic-pc/compile_dependencies.sh +++ b/machines/generic-pc/compile_dependencies.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e diff --git a/machines/generic-pc/get-julia.sh b/machines/generic-pc/get-julia.sh new file mode 100755 index 000000000..316bdf82b --- /dev/null +++ b/machines/generic-pc/get-julia.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Detect operating system, following +# https://stackoverflow.com/questions/3466166/how-to-check-if-running-in-cygwin-mac-or-linux +case "$(uname -s)" in + + Darwin*) + machines/shared/get-julia-macos.sh + ;; + + Linux*) + machines/shared/get-julia-linux-x86_64.sh + ;; + + CYGWIN*|MINGW*|MINGW32*|MSYS*) + echo "Setting up on Windows is unsupported" + exit 1 + OS=windows + ;; + + # Add here more strings to compare + # See correspondence table at the bottom of this answer + + *) + echo "Unrecognised operaing system, 'uname -s'= $(uname -s)" + exit 1 + ;; +esac + +exit 0 diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index bb7c6403e..ce6f4732c 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -547,7 +547,7 @@ if [[ $USE_PLOTS_POSTPROC -eq 0 ]]; then else # Re-write bin/julia to add activation of the Python venv LAST_LINES=$(tail -n 2 bin/julia) - echo '#!/bin/bash' > bin/julia # It is necessary to use single-quotes not double quotes here, but don't understand why + echo '#!/usr/bin/env bash' > bin/julia # It is necessary to use single-quotes not double quotes here, but don't understand why echo "source $PYTHON_VENV_PATH/bin/activate" >> bin/julia echo "$LAST_LINES" >> bin/julia fi diff --git a/machines/marconi/compile_dependencies.sh b/machines/marconi/compile_dependencies.sh index 55e4cbe8e..f47fad0cf 100755 --- a/machines/marconi/compile_dependencies.sh +++ b/machines/marconi/compile_dependencies.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e diff --git a/machines/marconi/get-julia.sh b/machines/marconi/get-julia.sh index 4562dfb77..95c1ad39d 100755 --- a/machines/marconi/get-julia.sh +++ b/machines/marconi/get-julia.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Make sure we have a sensible Python source machines/marconi/julia.env diff --git a/machines/marconi/jobscript-postprocess-plotsjl.template b/machines/marconi/jobscript-postprocess-plotsjl.template index af338535d..148c7bd3e 100644 --- a/machines/marconi/jobscript-postprocess-plotsjl.template +++ b/machines/marconi/jobscript-postprocess-plotsjl.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --ntasks=1 #SBATCH --time=POSTPROCTIME diff --git a/machines/marconi/jobscript-postprocess.template b/machines/marconi/jobscript-postprocess.template index c6f2e942e..9b4a5652f 100644 --- a/machines/marconi/jobscript-postprocess.template +++ b/machines/marconi/jobscript-postprocess.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --ntasks=1 #SBATCH --time=POSTPROCTIME diff --git a/machines/marconi/jobscript-precompile-makie-post-processing.template b/machines/marconi/jobscript-precompile-makie-post-processing.template index 75423540c..edbdb2c52 100644 --- a/machines/marconi/jobscript-precompile-makie-post-processing.template +++ b/machines/marconi/jobscript-precompile-makie-post-processing.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --ntasks=1 #SBATCH --time=1:00:00 diff --git a/machines/marconi/jobscript-precompile-no-run.template b/machines/marconi/jobscript-precompile-no-run.template index ee251c9eb..b47bb7d6a 100644 --- a/machines/marconi/jobscript-precompile-no-run.template +++ b/machines/marconi/jobscript-precompile-no-run.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --ntasks=1 #SBATCH --time=1:00:00 diff --git a/machines/marconi/jobscript-precompile-plots-post-processing.template b/machines/marconi/jobscript-precompile-plots-post-processing.template index 5234137fd..ff2ea1db0 100644 --- a/machines/marconi/jobscript-precompile-plots-post-processing.template +++ b/machines/marconi/jobscript-precompile-plots-post-processing.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --ntasks=1 #SBATCH --time=1:00:00 diff --git a/machines/marconi/jobscript-precompile.template b/machines/marconi/jobscript-precompile.template index 36190aaca..ae4cbd973 100644 --- a/machines/marconi/jobscript-precompile.template +++ b/machines/marconi/jobscript-precompile.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --ntasks=1 #SBATCH --time=1:00:00 diff --git a/machines/marconi/jobscript-restart.template b/machines/marconi/jobscript-restart.template index 420687c93..ffbcc746f 100644 --- a/machines/marconi/jobscript-restart.template +++ b/machines/marconi/jobscript-restart.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --nodes=NODES #SBATCH --cpus-per-task=1 diff --git a/machines/marconi/jobscript-run.template b/machines/marconi/jobscript-run.template index 34b189423..131dbbb6e 100644 --- a/machines/marconi/jobscript-run.template +++ b/machines/marconi/jobscript-run.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #SBATCH --nodes=NODES #SBATCH --cpus-per-task=1 diff --git a/machines/shared/get-julia-linux-x86_64.sh b/machines/shared/get-julia-linux-x86_64.sh index 8505b242a..cad7f2e9c 100755 --- a/machines/shared/get-julia-linux-x86_64.sh +++ b/machines/shared/get-julia-linux-x86_64.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ARTIFACT_DIR=machines/artifacts/ diff --git a/machines/shared/get-julia-macos.sh b/machines/shared/get-julia-macos.sh new file mode 100755 index 000000000..1caeec426 --- /dev/null +++ b/machines/shared/get-julia-macos.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +ARTIFACT_DIR=machines/artifacts/ + +ARCH="" +while [[ ! $ARCH -eq "x86_64" && ! $ARCH -eq "aarch64" ]]; do + echo "Which architecture does your Mac use - 'x86_64' (Intel processor)" + echo "or 'aarch64' (Apple processor)? x86_64/[aarch64]" + read -p "> " ARCH + if [[ -z $ARCH ]]; then + ARCH=aarch64 + fi +done + +# Download Julia binary distribution +if [ -z "$1" ]; then + # No version argument passed + JULIA_TAR=$(machines/shared/get-julia.py --output-dir $ARTIFACT_DIR --os mac --arch $ARCH) +else + JULIA_TAR=$(machines/shared/get-julia.py --output-dir $ARTIFACT_DIR --os mac --arch $ARCH --version $1) +fi + +cd $ARTIFACT_DIR + +# Get the directory name from the tar file, see +# https://unix.stackexchange.com/a/246807 +JULIA_DIR=$(tar tf $JULIA_TAR | head -n 1) + +# Assuming Julia's archives will always be gzipped (.tar.gz) here +tar -xzf $JULIA_TAR + +# echo the path to the Julia binary so we can use it in another script +echo $ARTIFACT_DIR/$JULIA_DIR/bin/julia + +exit 0 diff --git a/machines/shared/get-julia.py b/machines/shared/get-julia.py index 7741b3945..b5059ed03 100755 --- a/machines/shared/get-julia.py +++ b/machines/shared/get-julia.py @@ -97,6 +97,10 @@ def __str__(self): else: version = [x for x in version if "musl" not in x["triplet"]] + if args.os == "mac": + # Not sure what to do with .dmg version, so pick .tar.gz version + version = [x for x in version if x["extension"] == "tar.gz"] + if len(version) > 1: raise ValueError( f"Expected a single possibility for os={args.os} and arch={args.arch} with musl={args.musl}. Got {len(version)}." diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index c61126e32..450a3b292 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -186,7 +186,7 @@ function machine_setup_moment_kinetics(machine::String, # Make a script to run julia, including the JULIA_DEPOT_PATH so that we can avoid # needing the julia.env setup open(julia_executable_name, "w") do io - println(io, "#!/bin/bash") + println(io, "#!/usr/bin/env bash") println(io, "export JULIA_DEPOT_PATH=$julia_directory") julia_path = joinpath(Sys.BINDIR, "julia") println(io, "$julia_path \"\$@\"") diff --git a/machines/shared/precompile-makie-post-processing-submit.sh b/machines/shared/precompile-makie-post-processing-submit.sh index b40fbd093..73a2a3fc0 100755 --- a/machines/shared/precompile-makie-post-processing-submit.sh +++ b/machines/shared/precompile-makie-post-processing-submit.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e diff --git a/machines/shared/precompile-plots-post-processing-submit.sh b/machines/shared/precompile-plots-post-processing-submit.sh index d762b9131..f5d7e6b45 100755 --- a/machines/shared/precompile-plots-post-processing-submit.sh +++ b/machines/shared/precompile-plots-post-processing-submit.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e diff --git a/machines/shared/precompile-submit.sh b/machines/shared/precompile-submit.sh index 873ceb453..1b3e9c592 100755 --- a/machines/shared/precompile-submit.sh +++ b/machines/shared/precompile-submit.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e diff --git a/machines/shared/submit-restart.sh b/machines/shared/submit-restart.sh index 3186173a4..a1cf53afc 100755 --- a/machines/shared/submit-restart.sh +++ b/machines/shared/submit-restart.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e diff --git a/machines/shared/submit-run.sh b/machines/shared/submit-run.sh index b7251c775..f6fa2a6a6 100755 --- a/machines/shared/submit-run.sh +++ b/machines/shared/submit-run.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e From 6f1cc574a99671a2d0d4b414fed1bbcd9845dfdf Mon Sep 17 00:00:00 2001 From: John Omotani Date: Fri, 29 Dec 2023 16:26:03 +0000 Subject: [PATCH 32/80] Move most argument prompts into machine_setup.jl --- machines/machine_setup.sh | 305 ++----------------------------- machines/shared/machine_setup.jl | 304 ++++++++++++++---------------- 2 files changed, 156 insertions(+), 453 deletions(-) diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index ce6f4732c..218918fe7 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -94,24 +94,22 @@ else DOWNLOAD_JULIA=0 fi fi -if [[ $DOWNLOAD_JULIA -eq 0 ]]; then - # Need machine name to get the right get-julia.sh script. - # Not ideal to get input here as in the 'proper' place below we can check - # for a valid input for $MACHINE and list the allowed values (although - # there will be an error here if the input is not valid as - # 'machines/$MACHINE/get-julia.sh would not exist). However the checking - # requires Julia, so cannot do it here as we need the value in order to - # download Julia (need to know the machine so that we download the right - # set of binaries for the OS, architecture, etc. - while [ -z $MACHINE ]; do - echo "Enter name of the machine to set up [$DEFAULT_MACHINE]:" - read -p "> " MACHINE - echo - if [ -z $MACHINE ]; then - MACHINE=$DEFAULT_MACHINE - fi - done +# Get name of 'machine' +while [[ -z $MACHINE || !( $MACHINE == "generic-pc" || $MACHINE == "archer" || $MACHINE == "marconi" ) ]]; do + echo "Enter name of the machine to set up (must be one of 'generic-pc'," + echo "'archer', or 'marconi') [$DEFAULT_MACHINE]:" + read -p "> " MACHINE + echo + if [ -z $MACHINE ]; then + MACHINE=$DEFAULT_MACHINE + fi +done + +echo "Setting up for '$MACHINE'" +echo + +if [[ $DOWNLOAD_JULIA -eq 0 ]]; then # Download a version of Julia that is correct for this machine. # Here the user can specify which version of Julia they want to use in # case the latest stable versionis not wanted. @@ -171,223 +169,16 @@ echo "Using Julia at $JULIA" echo echo "$JULIA" > .julia_default.txt -# If Julia was downloaded by this script, $MACHINE is already set. Otherwise, -# need to get and check its value here. -if [ -z "$MACHINE" ]; then - # Make first attempt at getting the name of the machine to setup - echo "Enter name of the machine to set up [$DEFAULT_MACHINE]:" - read -p "> " MACHINE - echo - if [ -z $MACHINE ]; then - MACHINE=$DEFAULT_MACHINE - fi -fi -while true; do - # Get default values for this machine from the machine_setup.jl script - # Note: the brackets() around the command execution turn the result into an - # 'array' that we can get the separate elements from below. - # This command will fail (and print a helpful error message) if $MACHINE is - # not a known value, so `break` only when it succeeds. - DEFAULTS=($($JULIA machines/shared/machine_setup.jl --defaults $MACHINE)) && break - - # If we did not get a good value for $MACHINE yet, prompt for a new one. - echo "Enter name of the machine to set up:" - read -p "> " MACHINE - echo -done - -echo "Setting up for '$MACHINE'" -echo - # Save the machine name so we can use it as the default if we re-run # machine_setup.sh. echo $MACHINE > .this_machine_name.txt -if [[ $MACHINE -eq "generic-pc" ]]; then +if [[ $MACHINE == "generic-pc" ]]; then BATCH_SYSTEM=1 else BATCH_SYSTEM=0 fi -# Note that the $(eval echo )) is needed to remove quotes around -# arguments. Adding the quotes in the Julia script is necessary so that if an -# argument is empty it is not lost when parsing the Julia script output into -# $DEFAULTS. Note $DEFAULTS is a Bash array. -DEFAULT_RUN_TIME=$(eval echo ${DEFAULTS[0]}) -DEFAULT_NODES=$(eval echo ${DEFAULTS[1]}) -DEFAULT_POSTPROC_TIME=$(eval echo ${DEFAULTS[2]}) -DEFAULT_POSTPROC_MEMORY=$(eval echo ${DEFAULTS[3]}) -DEFAULT_PARTITION=$(eval echo ${DEFAULTS[4]}) -DEFAULT_QOS=$(eval echo ${DEFAULTS[5]}) -DEFAULT_ACCOUNT=$(eval echo ${DEFAULTS[6]}) -JULIA_DIRECTORY=$(eval echo ${DEFAULTS[7]}) -DEFAULT_USE_MAKIE_POSTPROC=$(eval echo ${DEFAULTS[8]}) -DEFAULT_USE_PLOTS_POSTPROC=$(eval echo ${DEFAULTS[9]}) -DEFAULT_USE_NETCDF=$(eval echo ${DEFAULTS[10]}) -DEFAULT_ENABLE_MMS=$(eval echo ${DEFAULTS[11]}) - -if [[ $DEFAULT_USE_MAKIE_POSTPROC -eq 0 ]]; then - echo "Would you like to set up makie_post_processing? [y]/n" - read -p "> " input - echo - while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - # $input must be empty, 'y' or 'n'. It is none of these, so ask for input - # again until we get a valid response. - echo - echo "$input is not a valid response: [y]/n" - read -p "> " input - echo - done - if [[ $input == "n" ]]; then - USE_MAKIE_POSTPROC=1 - else - USE_MAKIE_POSTPROC=0 - fi -else - echo "Would you like to set up makie_post_processing? y/[n]" - read -p "> " input - echo - while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - # $input must be empty, 'y' or 'n'. It is none of these, so ask for input - # again until we get a valid response. - echo - echo "$input is not a valid response: y/[n]" - read -p "> " input - echo - done - if [[ $input == "y" ]]; then - USE_MAKIE_POSTPROC=0 - else - USE_MAKIE_POSTPROC=1 - fi -fi - -if [[ $DEFAULT_USE_PLOTS_POSTPROC -eq 0 ]]; then - echo "Would you like to set up plots_post_processing? [y]/n" - read -p "> " input - echo - while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - # $input must be empty, 'y' or 'n'. It is none of these, so ask for input - # again until we get a valid response. - echo - echo "$input is not a valid response: [y]/n" - read -p "> " input - echo - done - if [[ $input == "n" ]]; then - USE_PLOTS_POSTPROC=1 - else - USE_PLOTS_POSTPROC=0 - fi -else - echo "Would you like to set up plots_post_processing? y/[n]" - read -p "> " input - echo - while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - # $input must be empty, 'y' or 'n'. It is none of these, so ask for input - # again until we get a valid response. - echo - echo "$input is not a valid response: y/[n]" - read -p "> " input - echo - done - if [[ $input == "y" ]]; then - USE_PLOTS_POSTPROC=0 - else - USE_PLOTS_POSTPROC=1 - fi -fi - - -if [[ $DEFAULT_USE_NETCDF -eq 0 ]]; then - echo "Would you like to enable optional NetCDF I/O (warning: using NetCDF sometimes" - echo "causes errors when using a local or system install of HDF5)? [y]/n" - read -p "> " input - echo - while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - # $input must be empty, 'y' or 'n'. It is none of these, so ask for input - # again until we get a valid response. - echo - echo "$input is not a valid response: [y]/n" - read -p "> " input - echo - done - if [[ $input == "n" ]]; then - USE_NETCDF=1 - else - USE_NETCDF=0 - fi -else - echo "Would you like to enable optional NetCDF I/O (warning: using NetCDF sometimes" - echo "causes errors when using a local or system install of HDF5)? y/[n]" - read -p "> " input - echo - while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - # $input must be empty, 'y' or 'n'. It is none of these, so ask for input - # again until we get a valid response. - echo - echo "$input is not a valid response: y/[n]" - read -p "> " input - echo - done - if [[ $input == "y" ]]; then - USE_NETCDF=0 - else - USE_NETCDF=1 - fi -fi - - -if [[ $DEFAULT_ENABLE_MMS -eq 0 ]]; then - echo "Would you like to enable MMS testing? [y]/n" - read -p "> " input - echo - while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - # $input must be empty, 'y' or 'n'. It is none of these, so ask for input - # again until we get a valid response. - echo - echo "$input is not a valid response: [y]/n" - read -p "> " input - echo - done - if [[ $input == "n" ]]; then - ENABLE_MMS=1 - else - ENABLE_MMS=0 - fi -else - echo "Would you like to to enable MMS testing? y/[n]" - read -p "> " input - echo - while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - # $input must be empty, 'y' or 'n'. It is none of these, so ask for input - # again until we get a valid response. - echo - echo "$input is not a valid response: y/[n]" - read -p "> " input - echo - done - if [[ $input == "y" ]]; then - ENABLE_MMS=0 - else - ENABLE_MMS=1 - fi -fi - -if [[ $BATCH_SYSTEM -eq 0 ]]; then - # Get the account to submit jobs with - echo "Enter the account code used to submit jobs [$DEFAULT_ACCOUNT]:" - read -p "> " ACCOUNT - if [[ -z ACCOUNT ]]; then - ACCOUNT=$DEFAULT_ACCOUNT - fi - echo - echo "Account code used is $ACCOUNT" - echo -else - ACCOUNT="" -fi - if [[ $BATCH_SYSTEM -eq 0 ]]; then # Batch systems can (conveniently) use different optimization flags for # running simulations and for post-processing. @@ -428,68 +219,6 @@ echo echo "Using julia_directory=$JULIA_DIRECTORY" echo -if [[ $BATCH_SYSTEM -eq 0 ]]; then - # Get the setting for the default run time - echo "Enter the default value for the time limit for simulation jobs [$DEFAULT_RUN_TIME]:" - read -p "> " input - if [ ! -z "$input" ]; then - DEFAULT_RUN_TIME=$input - fi - echo - echo "Default simulation time limit is $DEFAULT_RUN_TIME" - echo - - # Get the setting for the default number of nodes - echo "Enter the default value for the number of nodes for a run [$DEFAULT_NODES]:" - read -p "> " input - if [ ! -z "$input" ]; then - DEFAULT_NODES=$input - fi - echo - echo "Default number of nodes is $DEFAULT_NODES" - echo - - # Get the setting for the default postproc time - echo "Enter the default value for the time limit for post-processing jobs [$DEFAULT_POSTPROC_TIME]:" - read -p "> " input - if [ ! -z "$input" ]; then - DEFAULT_POSTPROC_TIME=$input - fi - echo - echo "Default post-processing time limit is $DEFAULT_POSTPROC_TIME" - echo - - # Get the setting for the default postproc memory - echo "Enter the default value for the memory requested for post-processing jobs [$DEFAULT_POSTPROC_MEMORY]:" - read -p "> " input - if [ ! -z "$input" ]; then - DEFAULT_POSTPROC_MEMORY=$input - fi - echo - echo "Default post-processing memory requested is $DEFAULT_POSTPROC_MEMORY" - echo - - # Get the setting for the default partition - echo "Enter the default value for the partition for simulation jobs [$DEFAULT_PARTITION]:" - read -p "> " input - if [ ! -z "$input" ]; then - DEFAULT_PARTITION=$input - fi - echo - echo "Default partiion for simulations is $DEFAULT_PARTITION" - echo - - # Get the setting for the default qos - echo "Enter the default value for the QOS for simulation jobs [$DEFAULT_QOS]:" - read -p "> " input - if [ ! -z "$input" ]; then - DEFAULT_QOS=$input - fi - echo - echo "Default QOS for simulations is $DEFAULT_QOS" - echo -fi - if [[ $BATCH_SYSTEM -eq 0 ]]; then echo "Do you want to submit a serial (or debug) job to precompile, creating the" echo "moment_kinetics.so image (this is required in order to use the job submission" @@ -523,7 +252,7 @@ echo # command, because passing as a prefix does not work (sometimes??) within a # bash script (even though as far as JTO knows it should work). export JULIA_DEPOT_PATH=$JULIA_DIRECTORY -$JULIA machines/shared/machine_setup.jl "$MACHINE" "$ACCOUNT" "$JULIA_DIRECTORY" "$DEFAULT_RUN_TIME" "$DEFAULT_NODES" "$DEFAULT_POSTPROC_TIME" "$DEFAULT_POSTPROC_MEMORY" "$DEFAULT_PARTITION" "$DEFAULT_QOS" "$USE_MAKIE_POSTPROC" "$USE_PLOTS_POSTPROC" "$USE_NETCDF" "$ENABLE_MMS" +$JULIA machines/shared/machine_setup.jl "$MACHINE" if [ -f julia.env ]; then # Set up modules, JULIA_DEPOT_PATH, etc. to use for the rest of this script diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index 450a3b292..8a53962ff 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -10,49 +10,95 @@ using TOML # Default settings for the arguments to machine_setup_moment_kinetics(), set like this # so that they can be passed to the bash script `machine_setup.sh` default_settings = Dict{String,Dict{String,String}}() -default_settings["base"] = Dict("default_run_time"=>"24:00:00", +default_settings["base"] = Dict("account"=>"", + "default_run_time"=>"24:00:00", "default_nodes"=>"1", "default_postproc_time"=>"1:00:00", "default_postproc_memory"=>"64G", "default_partition"=>"", "default_qos"=>"", - "use_makie"=>"1", - "use_plots"=>"1", - "use_netcdf"=>"1", - "enable_mms"=>"1") + "use_makie"=>"n", + "use_plots"=>"n", + "use_netcdf"=>"n", + "enable_mms"=>"n") # No batch system steup for "generic-pc" default_settings["generic-pc"] = merge(default_settings["base"], Dict("default_run_time"=>"0:00:00", "default_nodes"=>"0", "default_postproc_time"=>"0:00:00", "default_postproc_memory"=>"0", - "use_makie"=>"0")) + "use_makie"=>"y")) default_settings["archer"] = merge(default_settings["base"], Dict("default_partition"=>"standard", "default_qos"=>"standard")) default_settings["marconi"] = merge(default_settings["base"], Dict("default_partition"=>"skl_fua_prod", "default_qos"=>"normal")) +""" +""" +function get_user_input(possible_values, default_value) + setting = default_value + while true + if possible_values[1] == default_value + print("[", possible_values[1], "]") + else + print(possible_values[1]) + end + for x ∈ possible_values[2:end] + if x == default_value + print("/[", x, "]") + else + print("/$x") + end + end + println(":") + print("> ") + input = readline() + if input == "" + break + elseif input ∈ possible_values + setting = input + break + end + end + return setting +end + +""" + +Prompt the user to set a setting. Default value is read from LocalPreferences.toml if it +has been set before, or from sensible defaults otherwise. +""" +function get_setting(setting_name, message, machine, local_defaults, + possible_values=nothing) + # Get default value + default_value = get(local_defaults, setting_name, default_settings[machine][setting_name]) + + if possible_values === nothing + println("$message\n[$default_value]:") + print("> ") + setting = readline() + if setting == "" + setting = default_value + end + else + println(message) + setting = get_user_input(possible_values, default_value) + end + println("\nUsing $setting_name=$setting\n") + local_defaults[setting_name] = setting + + return setting +end """ - machine_setup_moment_kinetics(machine::String, - account::String, - julia_directory::String, - default_run_time::String, - default_nodes::String, - default_postproc_time::String, - default_postproc_memory::String, - default_partition::String; - default_qos::String; - use_makie::String; - use_plots::String; - use_netcdf::String; - enable_mms::String; - no_force_exit::Bool=false, + machine_setup_moment_kinetics(machine::String; ; no_force_exit::Bool=false, interactive::Bool=true) -Do setup for a known `machine`: +Do setup for a known `machine`, prompting the user for various settings (with defaults set +to sensible values - if the script has been run before, the defaults are the previously +used values): * On clusters that use a module system, provide `julia.env` at the top level of the moment_kinetics repo. @@ -64,8 +110,6 @@ Do setup for a known `machine`: get them for the current session) or in your `.bashrc` (to get them by default). Note that this calls `module purge` so will remove any currently loaded modules when it is run. -* Run setup commands for MPI and HDF5 which ensure the correct, system-provided - libraries are used. * Makes a symlink to the Julia exeutable used to run this command at `bin/julia` under the moment_kinetics repo, so that setup and job submission scripts can use a known relative path. @@ -74,49 +118,13 @@ Do setup for a known `machine`: to either replace the symlink `/bin/julia` by hand, or re-run this function using the new executable. -`julia_directory` gives the location of the directory (usually called `.julia`) where -Julia installs files, saves settings, etc. `julia_directory` must be passed if this -directory should be in a non-default location (i.e. not `\$HOME/.julia/`). The value is -used to set `JULIA_DEPOT_PATH` in the `julia.env` file or `bin/julia` script , so that -this setting is propagated to the environment on the compute nodes. - -Usually it is necessary for Julia to be restarted after running this function to ensure -the correct MPI is linked, etc. so the function will force Julia to exit. If for some -reason this is not desired (e.g. when debugging), pass `no_force_exit=true`. +Usually it is necessary for Julia to be restarted after running this function to run Julia +with the correct JULIA_DEPOT_PATH, etc. so the function will force Julia to exit. If for +some reason this is not desired (e.g. when debugging), pass `no_force_exit=true`. The `interactive` argument exists so that when this function is called from another script, terminal output with instructions for the next step can be disabled. -The remaining arguments can be used to change the default settings for jobs submitted -using the provided `submit-run.sh` script. These settings are read by the scripts from -`LocalPreferences.toml` and the values can safely be edited in that file without -re-running this function (if you want to). The arguments are: -* `default_run_time` is the maximum run time for the simulation, in the format expected - by `sbatch --time`, e.g. `"24:00:00"` for 24 hours, 0 minutes, 0 seconds. -* `default_nodes` is the default number of nodes to use for a simulation run. Note that - post-processing always runs in serial (using a serial or debug queue if available). -* `default_postproc_time` is the maximum run time for the post-processing job, in the - format expected by `sbatch --time`, e.g. `"1:00:00"` for 1 hours, 0 minutes, 0 - seconds. -* `default_postproc_memory` is the memory requested for the post-processing job, in the - format expected by `sbatch --mem`, e.g. `"64G"` for 64GB. -* `default_partition` is the default 'partition' passed to `sbatch --partition`. See your - cluster's documentation for possible values. The default will be the standard queue, - which charges towards the budget of your allocation. You might sometimes want, for - example, to change this to a debug queue if one is available. -* `default_qos` is the default 'quality of service' passed to `sbatch --qos`. See your - cluster's documentation for possible values. The default will be the standard queue, - which charges towards the budget of your allocation. You might want, for example, to - change this to a free, low-priority queue if one is available. -* `use_makie` indicates whether makie_post_processing has been enabled ("0" means yes, "1" - means no). -* `use_plots` indicates whether plots_post_processing has been enabled ("0" means yes, "1" - means no). -* `use_netcdf` indicates whether NetCDF I/O has been enabled ("0" means yes, "1" means - no). -* `enable_mms` indicates whether MMS testing has been enabled ("0" means yes, "1" means - no). - Currently supported machines: * `"generic-pc"` - A generic personal computer. Set up for interactive use, rather than for submitting jobs to a batch queue. @@ -129,24 +137,26 @@ Currently supported machines: `Preferences.jl` package). It might sometimes be useful to edit these by hand (e.g. the `account` setting if this needs to be changed.): it is fine to do this. """ -function machine_setup_moment_kinetics(machine::String, - account::String, - julia_directory::String, - default_run_time::String, - default_nodes::String, - default_postproc_time::String, - default_postproc_memory::String, - default_partition::String, - default_qos::String, - use_makie::String, - use_plots::String, - use_netcdf::String, - enable_mms::String; - no_force_exit::Bool=false, +function machine_setup_moment_kinetics(machine::String; no_force_exit::Bool=false, interactive::Bool=true) repo_dir = dirname(dirname(dirname(@__FILE__))) + # Get defaults from LocalPreferences.toml if possible + if isfile("LocalPreferences.toml") + local_preferences = TOML.parsefile("LocalPreferences.toml") + if "moment_kinetics" ∈ keys(local_preferences) + mk_preferences = local_preferences["moment_kinetics"] + else + mk_preferences = local_preferences["moment_kinetics"] = Dict{String,Any}() + end + else + local_preferences = Dict{String,Any}() + mk_preferences = local_preferences["moment_kinetics"] = Dict{String,Any}() + end + + mk_preferences["machine"] = machine + # Common operations that only depend on the name of `machine` ############################################################# @@ -156,6 +166,56 @@ function machine_setup_moment_kinetics(machine::String, batch_system = true end + # Get some settings + julia_directory = mk_preferences["julia_directory"] = ENV["JULIA_DEPOT_PATH"] + if batch_system + get_setting("default_run_time", + "Enter the default value for the time limit for simulation jobs", + machine, mk_preferences) + get_setting("default_nodes", + "Enter the default value for the number of nodes for a run", + machine, mk_preferences) + get_setting("default_postproc_time", + "Enter the default value for the time limit for post-processing jobs", + machine, mk_preferences) + get_setting("default_postproc_memory", + "Enter the default value for the memory requested for post-processing jobs", + machine, mk_preferences) + get_setting("default_partition", + "Enter the default value for the partition for simulation jobs", + machine, mk_preferences) + get_setting("default_qos", + "Enter the default value for the QOS for simulation jobs", + machine, mk_preferences) + get_setting("account", + "Enter the account code used to submit jobs", + machine, mk_preferences) + end + get_setting("use_makie", + "Would you like to set up makie_post_processing?", + machine, mk_preferences, ["y", "n"]) + get_setting("use_plots", + "Would you like to set up plots_post_processing?", + machine, mk_preferences, ["y", "n"]) + get_setting("use_netcdf", + "Would you like to enable optional NetCDF I/O (warning: using NetCDF sometimes\n" + * "causes errors when using a local or system install of HDF5)?", + machine, mk_preferences, ["y", "n"]) + get_setting("enable_mms", + "Would you like to enable MMS testing?", + machine, mk_preferences, ["y", "n"]) + + # Write these preferences into a [moment_kinetics] section in LocalPreferences.toml + # + # Load and re-write LocalPreferences.toml directly here to avoid needing to import + # the Preferences.jl package, which would need to be installed (TOML.jl is available + # as part of the Julia system). This is a bit hacky, but hopefully no need to do + # anything fancy here! + println("\n** Adding system-specific settings for moment_kinetics to LocalPreferences.toml\n") + open("LocalPreferences.toml", "w") do io + TOML.print(io, local_preferences, sorted=true) + end + if batch_system # Only use julia.env for a batch system as for an interactive system there are no # modules to set up and source'ing julia.env is mildly inconvenient. @@ -211,38 +271,6 @@ function machine_setup_moment_kinetics(machine::String, make_executable!(julia_executable_name) end - # Write these preferences into a [moment_kinetics] section in LocalPreferences.toml - # - # Load and re-write LocalPreferences.toml directly here to avoid needing to import - # the Preferences.jl package, which would need to be installed (TOML.jl is available - # as part of the Julia system). This is a bit hacky, but hopefully no need to do - # anything fancy here! - println("\n** Adding system-specific settings for moment_kinetics to LocalPreferences.toml\n") - local_preferences_filename = joinpath(repo_dir, "LocalPreferences.toml") - if ispath(local_preferences_filename) - local_preferences = TOML.parsefile(local_preferences_filename) - else - local_preferences = Dict{String,Any}() - end - mk_preferences = get(local_preferences, "moment_kinetics", Dict{String,String}()) - local_preferences["moment_kinetics"] = mk_preferences - mk_preferences["machine"] = machine - mk_preferences["julia_directory"] = julia_directory - mk_preferences["default_run_time"] = default_run_time - mk_preferences["default_nodes"] = default_nodes - mk_preferences["default_postproc_time"] = default_postproc_time - mk_preferences["default_postproc_memory"] = default_postproc_memory - mk_preferences["default_partition"] = default_partition - mk_preferences["default_qos"] = default_qos - mk_preferences["account"] = account - mk_preferences["use_makie"] = use_makie - mk_preferences["use_plots"] = use_plots - mk_preferences["use_netcdf"] = use_netcdf - mk_preferences["enable_mms"] = enable_mms - open(local_preferences_filename, "w") do io - TOML.print(io, local_preferences, sorted=true) - end - # If it is necessary to run a shell script to compile dependencies, set # this flag to true. compile_dependencies_relative_path = joinpath("machines", "shared", @@ -276,7 +304,7 @@ function machine_setup_moment_kinetics(machine::String, error("Unsupported machine '$machine'") end - if needs_account && account == "" + if needs_account && mk_preferences["account"] == "" error("For machine=\"$machine\" it is required to pass a value for the " * "`account` argument.") end @@ -316,7 +344,6 @@ end end using .machine_setup -using TOML if abspath(PROGRAM_FILE) == @__FILE__ # Allow the command to be called as a script. @@ -326,66 +353,13 @@ if abspath(PROGRAM_FILE) == @__FILE__ if "-h" ∈ ARGS || "--help" ∈ ARGS println("Script to set up moment_kinetics to run on a cluster.") println() - println("The settings requested correspond to the arguments to ") - println("`machine_setup_moment_kinetics()`. See the function docstring:") + println("Runs `machine_setup_moment_kinetics()`. See the function docstring:") println() println(@doc machine_setup_moment_kinetics) exit(9) - elseif "-d" ∈ ARGS || "--defaults" ∈ ARGS - # Print out the default values for arguments for this machine - i = "-d" ∈ ARGS ? findfirst(x -> x=="-d", ARGS) : - findfirst(x -> x=="--defaults", ARGS) - if length(ARGS) <= i - println("Must pass a machine name after `-d` or `--defaults`") - exit(1) - end - machine = ARGS[i+1] - - known_machines = sort(collect(keys(machine_setup.default_settings))) - # Remove "base" which is a dummy entry - known_machines = filter(x -> x ≠ "base", known_machines) - if !(machine ∈ keys(machine_setup.default_settings)) - println(stderr, "Error: machine \"$machine\" not recognised") - println(stderr, " Known machines are $known_machines\n") - exit(1) - end - - d = deepcopy(machine_setup.default_settings[machine]) - # default setting for "julia_directory" is the JULIA_DEPOT_PATH environment - # variable - d["julia_directory"] = get(ENV, "JULIA_DEPOT_PATH", "") - # No default for "account". - d["account"] = "" - - # If settings have already been saved (i.e. machine_setup.sh has already been - # run), then use the previous settings as the default this time. - # Use TOML.parsefile() to read the existing preferences to avoid depending on the - # Preferences package at this point (because we might want to set - # JULIA_DEPOT_PATH, but not have set it yet). - if ispath("LocalPreferences.toml") - existing_settings = get(TOML.parsefile("LocalPreferences.toml"), - "moment_kinetics", Dict{String, String}()) - else - existing_settings = Dict{String, String}() - end - for setting ∈ ("default_run_time", "default_nodes", "default_postproc_time", - "default_postproc_memory", "default_partition", - "default_partition", "account", "julia_directory", "use_makie", - "use_plots", "use_netcdf", "enable_mms") - d[setting] = get(existing_settings, setting, d[setting]) - end - - println("\"", d["default_run_time"], "\" \"",d["default_nodes"], "\" \"", - d["default_postproc_time"], "\" \"", d["default_postproc_memory"], - "\" \"", d["default_partition"], "\" \"", d["default_qos"], "\" \"", - d["account"], "\" \"", d["julia_directory"], "\" \"", d["use_makie"], - "\" \"", d["use_plots"], "\" \"", d["use_netcdf"], "\" \"", - d["enable_mms"], "\"") - exit(0) end - # Get function arguments from ARGS - machine_setup_moment_kinetics(ARGS...; interactive=false) + machine_setup_moment_kinetics(ARGS[1]; interactive=false) exit(0) end From d01fe93087d6fa6d82238f7a00fc659ffdc0b1f2 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 30 Dec 2023 11:08:23 +0000 Subject: [PATCH 33/80] Save the setting for JULIA_DIRECTORY and use it as the default on re-run --- .gitignore | 1 + machines/machine_setup.sh | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index d09eeea92..4de5cd7f4 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ machines/artifacts /submit-restart.sh /submit-run.sh /.julia_default.txt +/.julia_directory_default.txt /.this_machine_name.txt moment_kinetics.so precompile-temp diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 218918fe7..1bdb9616c 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -195,6 +195,14 @@ fi # Get the location for the .julia directory, in case this has to have a # non-default value, e.g. because the user's home directory is not accessible # from compute nodes. +if [[ -f .julia_directory_default.txt ]]; then + JULIA_DIRECTORY=$(cat .julia_directory_default.txt) +else + # If we do not have an existing setting, try using $JULIA_DEPOT_PATH (if that + # is not set, then we end up with an empty default, which means use the + # standard default location). + JULIA_DIRECTORY=$JULIA_DEPOT_PATH +fi echo "It can be useful or necessary to set a non-default location for the " echo ".julia directory. Leave this empty if the default location is OK." echo "Enter a name for a subdirectory of the current directory, e.g. " @@ -218,6 +226,7 @@ fi echo echo "Using julia_directory=$JULIA_DIRECTORY" echo +echo $JULIA_DIRECTORY > .julia_directory_default.txt if [[ $BATCH_SYSTEM -eq 0 ]]; then echo "Do you want to submit a serial (or debug) job to precompile, creating the" From dd8301e4f5bf7dbb5f4d061030f4c88ea5f74462 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 30 Dec 2023 11:45:10 +0000 Subject: [PATCH 34/80] Start moving more setup into machine_setup_stage_two.jl --- machines/machine_setup.sh | 88 +-------------------- machines/shared/machine_setup.jl | 6 ++ machines/shared/machine_setup_stage_two.jl | 90 +++++++++++++++++++--- 3 files changed, 87 insertions(+), 97 deletions(-) diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 1bdb9616c..e18232daa 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -228,28 +228,6 @@ echo "Using julia_directory=$JULIA_DIRECTORY" echo echo $JULIA_DIRECTORY > .julia_directory_default.txt -if [[ $BATCH_SYSTEM -eq 0 ]]; then - echo "Do you want to submit a serial (or debug) job to precompile, creating the" - echo "moment_kinetics.so image (this is required in order to use the job submission" - echo "scripts and templates provided)? [y]/n:" - read -p "> " input - - while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do - # $input must be empty, 'y' or 'n'. It is none of these, so ask for input - # again until we get a valid response. - echo - echo "$input is not a valid response: [y]/n" - read -p "> " input - done - if [[ -z $input || $input == "y" ]]; then - SUBMIT_PRECOMPILATION=0 - else - SUBMIT_PRECOMPILATION=1 - fi -else - SUBMIT_PRECOMPILATION=1 -fi - # Now we have a 'julia' executable and the settings, can call a Julia script # (machines/shared/machine_setup.jl) to create LocalPreferences.toml, # julia.env, bin/julia, and some machine-specific symlinks. @@ -268,29 +246,6 @@ if [ -f julia.env ]; then source julia.env fi -if [[ $USE_PLOTS_POSTPROC -eq 0 ]]; then - # Create a Python venv, ensure it contains matplotlib, and append its - # activation command to julia.env. - # Use the `--system-site-packages` option to let the venv include any packages - # already installed by the system. - PYTHON_VENV_PATH=$PWD/machines/artifacts/mk_venv - python -m venv --system-site-packages $PYTHON_VENV_PATH - source $PYTHON_VENV_PATH/bin/activate - # Use 'PYTHONNOUSERSITE=1' so that pip ignores any packages in ~/.local (which - # may not be accessible from compute nodes on some clusters) and therefore - # definitely installs matplotlib and its dependencies into mk_venv. - PYTHONNOUSERSITE=1 pip install matplotlib - if [[ $BATCH_SYSTEM -eq 0 ]]; then - echo "source $PYTHON_VENV_PATH/bin/activate" >> julia.env - else - # Re-write bin/julia to add activation of the Python venv - LAST_LINES=$(tail -n 2 bin/julia) - echo '#!/usr/bin/env bash' > bin/julia # It is necessary to use single-quotes not double quotes here, but don't understand why - echo "source $PYTHON_VENV_PATH/bin/activate" >> bin/julia - echo "$LAST_LINES" >> bin/julia - fi -fi - # [ -f ] tests if exists and is a file if [ -f machines/shared/compile_dependencies.sh ]; then # Need to compile some dependencies @@ -299,33 +254,9 @@ if [ -f machines/shared/compile_dependencies.sh ]; then machines/shared/compile_dependencies.sh fi -# We want to always add a couple of dependencies that are required to run the -# tests in the top-level environment by just 'include()'ing the test scripts. -# We also run the 'stage two' setup now if it is required. -SETUP_COMMAND="bin/julia --project $OPTIMIZATION_FLAGS -e 'import Pkg" -if [[ $USE_NETCDF -eq 1 || $ENABLE_MMS -eq 1 ]]; then - # Remove packages used by non-selected extensions in case they were installed previously - if [[ $USE_NETCDF -eq 1 ]]; then - SETUP_COMMAND="$SETUP_COMMAND; try Pkg.rm(\"NCDatasets\") catch end" - fi - if [[ $ENABLE_MMS -eq 1 ]]; then - SETUP_COMMAND="$SETUP_COMMAND; try Pkg.rm([\"Symbolics\", \"IfElse\"]) catch end" - fi -fi -SETUP_COMMAND="$SETUP_COMMAND; Pkg.add([\"HDF5\", \"MPI\", \"MPIPreferences\", \"SpecialFunctions\"" -if [[ $USE_NETCDF -eq 0 ]]; then - # Install NetCDF interface package NCDatasets to enable file_io_netcdf extension - SETUP_COMMAND="$SETUP_COMMAND, \"NCDatasets\"" -fi -if [[ $ENABLE_MMS -eq 0 ]]; then - # Install Symbolics and IfElse packages required by manufactured_solns_ext extension - SETUP_COMMAND="$SETUP_COMMAND, \"Symbolics\", \"IfElse\"" -fi -SETUP_COMMAND="$SETUP_COMMAND]); include(\"machines/shared/machine_setup_stage_two.jl\")'" -eval "$SETUP_COMMAND" - -# Add moment_kinetics package to the working project -bin/julia --project $OPTIMIZATION_FLAGS -e 'import Pkg; Pkg.develop(path="moment_kinetics"); Pkg.precompile()' +# Don't use bin/julia for machine_setup_stage_two.jl because that script modifies bin/julia. +# It is OK to not use it here, because JULIA_DEPOT_PATH has been set within this script +$JULIA --project $OPTIMIZATION_FLAGS machines/shared/machine_setup_stage_two.jl if [[ $USE_MAKIE_POSTPROC -eq 0 ]]; then echo "Setting up makie_post_processing" @@ -353,19 +284,6 @@ else fi fi -if [[ $BATCH_SYSTEM -eq 0 ]]; then - # Make symlinks to submission scripts - ln -s machines/shared/precompile-submit.sh - ln -s machines/shared/submit-run.sh - ln -s machines/shared/submit-restart.sh - if [[ $USE_MAKIE_POSTPROC -eq 0 ]]; then - ln -s machines/shared/precompile-makie-post-processing-submit.sh - fi - if [[ $USE_PLOTS_POSTPROC -eq 0 ]]; then - ln -s machines/shared/precompile-plots-post-processing-submit.sh - fi -fi - if [[ $SUBMIT_PRECOMPILATION -eq 0 ]]; then # This script launches a job that runs precompile.jl to create the # moment_kinetics.so image. diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index 8a53962ff..f1112fd81 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -17,6 +17,7 @@ default_settings["base"] = Dict("account"=>"", "default_postproc_memory"=>"64G", "default_partition"=>"", "default_qos"=>"", + "submit_precompilation"=>"y", "use_makie"=>"n", "use_plots"=>"n", "use_netcdf"=>"n", @@ -190,6 +191,11 @@ function machine_setup_moment_kinetics(machine::String; no_force_exit::Bool=fals get_setting("account", "Enter the account code used to submit jobs", machine, mk_preferences) + get_setting("submit_precompilation", + "Do you want to submit a serial (or debug) job to precompile, creating the\n" + * "moment_kinetics.so image (this is required in order to use the job submission\n" + * "scripts and templates provided)?\n", + machine, mk_preferences, ["y", "n"]) end get_setting("use_makie", "Would you like to set up makie_post_processing?", diff --git a/machines/shared/machine_setup_stage_two.jl b/machines/shared/machine_setup_stage_two.jl index 8c030c90a..7d8686222 100644 --- a/machines/shared/machine_setup_stage_two.jl +++ b/machines/shared/machine_setup_stage_two.jl @@ -33,8 +33,9 @@ end repo_dir = dirname(dirname(dirname(@__FILE__))) -local_settings = TOML.parsefile(joinpath(repo_dir, "LocalPreferences.toml"))["moment_kinetics"] -machine = local_settings["machine"] +local_preferences = TOML.parsefile(joinpath(repo_dir, "LocalPreferences.toml")) +mk_preferences = local_preferences["moment_kinetics"] +machine = mk_preferences["machine"] machine_dir = joinpath(repo_dir, "machines", machine) machine_settings_filename = joinpath(machine_dir, "machine_settings.toml") if isfile(machine_settings_filename) @@ -43,6 +44,55 @@ else machine_settings = Dict{String,Any}() end +if machine == "generic-pc" + batch_system = false +else + batch_system = true +end + +if mk_preferences["use_plots"] == "y" + python_venv_path = joinpath(repo_dir, "machines", "artifacts", "mk_venv") + activate_path = joinpath(python_venv_path, "bin", "activate") + run(`bash -c "python -m venv --system-site-packages $python_venv_path; source $activate_path; PYTHONNOUSERSITE=1 pip install matplotlib"`) + if batch_system + open("julia.env", "a") do io + println(io, "source $activate_path") + end + else + bin_path = joinpath(repo_dir, "bin", "julia") + contents = readlines(bin_path) + open(bin_path, "w") do io + println(io, contents[1]) + println(io, "source $activate_path") + for line ∈ contents[2:end] + println(io, line) + end + end + end +end + +to_add = String["HDF5", "MPI", "MPIPreferences", "SpecialFunctions"] +to_rm = String[] +if mk_preferences["use_netcdf"] == "y" + push!(to_add, "NCDatasets") +else + push!(to_rm, "NCDatasets") +end +if mk_preferences["enable_mms"] == "y" + push!(to_add, "Symbolics", "IfElse") +else + push!(to_rm, "Symbolics", "IfElse") +end +for p ∈ to_rm + # If `p` was not previously installed, then Pkg.rm(p) with throw an error. We only + # need to remove if p was previously installed, so it is OK to ignore the error. + try + Pkg.rm(p) + catch + end +end +Pkg.add(to_add) + # Instantiate packages so we can use MPIPreferences below ######################################################### @@ -98,16 +148,6 @@ elseif machine_settings["hdf5_library_setting"] == "prompt" default_hdf5_dir = get(ENV, "HDF5_DIR", "") # try to find a path to a system hdf5, may not work on all systems - using TOML - repo_dir = dirname(dirname(dirname(@__FILE__))) - local_preferences_filename = joinpath(repo_dir, "LocalPreferences.toml") - if ispath(local_preferences_filename) - local_preferences = TOML.parsefile(local_preferences_filename) - else - local_preferences = Dict{String,Any}() - end - mk_preferences = get(local_preferences, "moment_kinetics", Dict{String,String}()) - println("mk_preferences ", mk_preferences) default_hdf5_dir = get(mk_preferences, "hdf5_dir", default_hdf5_dir) hdf5_dir = "" @@ -167,6 +207,32 @@ else end +Pkg.develop(path="moment_kinetics") +Pkg.precompile() + + +if batch_system + # Make symlinks to batch job submission scripts + symlink("precompile-submit.sh", joinpath("machines", "shared", "precompile-submit.sh")) + symlink("submit-run.sh", joinpath("machines", "shared", "submit-run.sh")) + symlink("submit-restart.sh", joinpath("machines", "shared", "submit-restart.sh")) + if mk_preferences["use_makie"] + symlink("precompile-makie-post-processing-submit.sh", + joinpath("machines", "shared", + "precompile-makie-post-processing-submit.sh")) + end + if mk_preferences["use_plots"] + symlink("precompile-plots-post-processing-submit.sh", + joinpath("machines", "shared", + "precompile-plots-post-processing-submit.sh")) + end + + if mk_preferences["submit_precompilation"] == "y" + run(`precompile-submit.sh`) + end +end + + # Force exit so Julia must be restarted ####################################### From 01fac0b923da4b83f98b225b3669b3b3733cf592 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 30 Dec 2023 12:39:42 +0000 Subject: [PATCH 35/80] Save batch_system flag --- machines/shared/machine_setup.jl | 1 + machines/shared/machine_setup_stage_two.jl | 6 +----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index f1112fd81..a436eeae1 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -166,6 +166,7 @@ function machine_setup_moment_kinetics(machine::String; no_force_exit::Bool=fals else batch_system = true end + mk_preferences["batch_system"] = batch_system # Get some settings julia_directory = mk_preferences["julia_directory"] = ENV["JULIA_DEPOT_PATH"] diff --git a/machines/shared/machine_setup_stage_two.jl b/machines/shared/machine_setup_stage_two.jl index 7d8686222..3fdc3b091 100644 --- a/machines/shared/machine_setup_stage_two.jl +++ b/machines/shared/machine_setup_stage_two.jl @@ -44,11 +44,7 @@ else machine_settings = Dict{String,Any}() end -if machine == "generic-pc" - batch_system = false -else - batch_system = true -end +batch_system = mk_preferences["batch_system"] if mk_preferences["use_plots"] == "y" python_venv_path = joinpath(repo_dir, "machines", "artifacts", "mk_venv") From 246f1e1e0dfea6931f7b514840d9f617f58a1e93 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 30 Dec 2023 12:40:13 +0000 Subject: [PATCH 36/80] Refactor setup of post processing packages --- machines/machine_setup.sh | 42 +--- .../shared/add_dependencies_to_project.jl | 205 ++++++++++++++++++ machines/shared/machine_setup_stage_two.jl | 177 +-------------- .../shared/makie_post_processing_setup.jl | 46 ++++ .../shared/plots_post_processing_setup.jl | 42 ++++ 5 files changed, 298 insertions(+), 214 deletions(-) create mode 100644 machines/shared/add_dependencies_to_project.jl create mode 100644 machines/shared/makie_post_processing_setup.jl create mode 100644 machines/shared/plots_post_processing_setup.jl diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index e18232daa..277f3b8bb 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -254,48 +254,12 @@ if [ -f machines/shared/compile_dependencies.sh ]; then machines/shared/compile_dependencies.sh fi +bin/julia --project $OPTIMIZATION_FLAGS machines/shared/add_dependencies_to_project.jl # Don't use bin/julia for machine_setup_stage_two.jl because that script modifies bin/julia. # It is OK to not use it here, because JULIA_DEPOT_PATH has been set within this script $JULIA --project $OPTIMIZATION_FLAGS machines/shared/machine_setup_stage_two.jl - -if [[ $USE_MAKIE_POSTPROC -eq 0 ]]; then - echo "Setting up makie_post_processing" - if [[ $BATCH_SYSTEM -eq 0 ]]; then - bin/julia --project=makie_post_processing/ $POSTPROC_OPTIMIZATION_FLAGS -e 'import Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.develop(path="makie_post_processing/makie_post_processing"); Pkg.precompile()' - else - bin/julia --project -e $OPTIMIZATION_FLAGS 'import Pkg; Pkg.develop(path="makie_post_processing/makie_post_processing")' - fi -else - if [[ $BATCH_SYSTEM -eq 1 ]]; then - bin/julia --project $OPTIMIZATION_FLAGS -e 'import Pkg; try Pkg.rm("makie_post_processing") catch end' - fi -fi - -if [[ $USE_PLOTS_POSTPROC -eq 0 ]]; then - echo "Setting up plots_post_processing" - if [[ $BATCH_SYSTEM -eq 0 ]]; then - bin/julia --project=plots_post_processing/ $POSTPROC_OPTIMIZATION_FLAGS -e 'import Pkg; Pkg.develop(path="moment_kinetics/"); Pkg.develop(path="plots_post_processing/plots_post_processing"); Pkg.precompile()' - else - bin/julia --project $OPTIMIZATION_FLAGS -e 'import Pkg; Pkg.develop(path="plots_post_processing/plots_post_processing")' - fi -else - if [[ $BATCH_SYSTEM -eq 1 ]]; then - bin/julia --project $OPTIMIZATION_FLAGS -e 'import Pkg; try Pkg.rm("plots_post_processing") catch end' - fi -fi - -if [[ $SUBMIT_PRECOMPILATION -eq 0 ]]; then - # This script launches a job that runs precompile.jl to create the - # moment_kinetics.so image. - ./precompile-submit.sh - - if [[ $USE_MAKIE_POSTPROC -eq 0 ]]; then - ./precompile-makie-post-processing-submit.sh - fi - if [[ $USE_PLOTS_POSTPROC -eq 0 ]]; then - ./precompile-plots-post-processing-submit.sh - fi -fi +bin/julia --project $POSTPROC_OPTIMIZATION_FLAGS machines/shared/makie_post_processing_setup.jl +bin/julia --project $POSTPROC_OPTIMIZATION_FLAGS machines/shared/plots_post_processing_setup.jl echo echo "Finished!" diff --git a/machines/shared/add_dependencies_to_project.jl b/machines/shared/add_dependencies_to_project.jl new file mode 100644 index 000000000..3b9cae826 --- /dev/null +++ b/machines/shared/add_dependencies_to_project.jl @@ -0,0 +1,205 @@ +using Pkg, TOML + +if abspath(PROGRAM_FILE) == @__FILE__ + prompt_for_hdf5 = true + repo_dir = dirname(dirname(dirname(@__FILE__))) + local_preferences_filename = joinpath(repo_dir, "LocalPreferences.toml") + local_preferences = TOML.parsefile(local_preferences_filename) + mk_preferences = local_preferences["moment_kinetics"] +else + prompt_for_hdf5 = false +end + +machine = mk_preferences["machine"] +machine_dir = joinpath(repo_dir, "machines", machine) +machine_settings_filename = joinpath(machine_dir, "machine_settings.toml") +if isfile(machine_settings_filename) + machine_settings = TOML.parsefile(machine_settings_filename) +else + machine_settings = Dict{String,Any}() +end + + +""" + get_input_with_path_completion(message=nothing) + +Print `message` and get user input using the `read` utility from bash to allow +path-completion. + +Solution adapted from +https://discourse.julialang.org/t/collecting-all-output-from-shell-commands/15592/2 +""" +function get_input_with_path_completion(message=nothing) + if message !== nothing + println(message) + end + + # The `out` object is used by `pipeline()` to capture output from the shell command. + out = Pipe() + + # Use Julia's shell command functionality to actually run bash - not sure how to use + # `read` and get output from it without using bash. + run(pipeline(`bash -c "read -e -p '> ' USERINPUT; echo \$USERINPUT"`, stdout=out)) + + # Need to close `out.in` to be able to read from `out`. + close(out.in) + + # chomp removes the trailing '\n'. 'String()' converts Vector{Char} to a String. + input = chomp(String(read(out))) + + return input +end + + +to_add = String["HDF5", "MPI", "MPIPreferences", "SpecialFunctions"] +to_rm = String[] +if mk_preferences["use_netcdf"] == "y" + push!(to_add, "NCDatasets") +else + push!(to_rm, "NCDatasets") +end +if mk_preferences["enable_mms"] == "y" + push!(to_add, "Symbolics", "IfElse") +else + push!(to_rm, "Symbolics", "IfElse") +end +for p ∈ to_rm + # If `p` was not previously installed, then Pkg.rm(p) with throw an error. We only + # need to remove if p was previously installed, so it is OK to ignore the error. + try + Pkg.rm(p) + catch + end +end +Pkg.add(to_add) + + +# Instantiate packages so we can use MPIPreferences below +######################################################### + +Pkg.instantiate() +Pkg.resolve() + + +# MPI setup +########### + +println("\n** Setting up to use system MPI\n") +using MPIPreferences + +if "mpi_library_names" ∈ keys(machine_settings) || "mpiexec" ∈ keys(machine_settings) + MPIPreferences.use_system_binary(library_names=machine_settings["mpi_library_names"], + mpiexec=machine_settings["mpiexec"]) +else + # If settings for MPI library are not given explicitly, then auto-detection by + # MPIPreferences.use_system_binary() should work. + MPIPreferences.use_system_binary() +end + + +# HDF5 setup +############ + +println("\n** Setting up to use system HDF5\n") + +if machine_settings["hdf5_library_setting"] == "system" + hdf5_dir = ENV["HDF5_DIR"] # system hdf5 + using HDF5 + HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), + joinpath(hdf5_dir, "libhdf5_hl.so")) +elseif machine_settings["hdf5_library_setting"] == "download" + artifact_dir = joinpath(repo_dir, "machines", "artifacts") + hdf5_dir = joinpath(artifact_dir, "hdf5-build", "lib") + using HDF5 + HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), + joinpath(hdf5_dir, "libhdf5_hl.so")) +elseif machine_settings["hdf5_library_setting"] == "prompt" + # Prompt user to select what HDF5 to use + if mk_preferences["build_hdf5"] == "0" + local_hdf5_install_dir = joinpath("machines", "artifacts", "hdf5-build", "lib") + local_hdf5_install_dir = realpath(local_hdf5_install_dir) + # We have downloaded and compiled HDF5, so link that + hdf5_dir = local_hdf5_install_dir + hdf5_lib = joinpath(local_hdf5_install_dir, "libhdf5.so") + hdf5_lib_hl = joinpath(local_hdf5_install_dir, "libhdf5_hl.so") + elseif !prompt_for_hdf5 + hdf5_dir = mk_preferences("hdf5_dir") + if hdf5_dir != "default" + hdf5_lib = joinpath(hdf5_dir, "libhdf5.so") + hdf5_lib_hl = joinpath(hdf5_dir, "libhdf5_hl.so") + end + else + println("\n** Setting up to use system HDF5\n") + + default_hdf5_dir = get(ENV, "HDF5_DIR", "") # try to find a path to a system hdf5, may not work on all systems + + default_hdf5_dir = get(mk_preferences, "hdf5_dir", default_hdf5_dir) + + hdf5_dir = "" + hdf5_lib = "" + hdf5_lib_hl = "" + while true + global hdf5_dir, hdf5_lib, hdf5_lib_hl + hdf5_dir = get_input_with_path_completion( + "\nAn HDF5 installation compiled with your system MPI is required to use\n" + * "parallel I/O. Enter the directory where the libhdf5.so and libhdf5_hl.so are\n" + * "located (enter 'default' to use the Julia-provided HDF5, which does not\n" + * "support parallel I/O): [$default_hdf5_dir]") + + if hdf5_dir == "" + hdf5_dir = default_hdf5_dir + end + + if hdf5_dir == "default" + break + end + + if isdir(hdf5_dir) + hdf5_dir = realpath(hdf5_dir) + end + hdf5_lib = joinpath(hdf5_dir, "libhdf5.so") + hdf5_lib_hl = joinpath(hdf5_dir, "libhdf5_hl.so") + if isfile(hdf5_lib) && isfile(hdf5_lib_hl) + break + else + # Remove trailing slash if it exists so that we can print a single trailing slash + # consistently + hdf5_dir = rstrip(hdf5_dir, '/') + print("HDF5 libraries not found in '$hdf5_dir/'.") + if !isfile(hdf5_lib) + print(" $hdf5_lib does not exist.") + end + if !isfile(hdf5_lib_hl) + print(" $hdf5_lib_hl does not exist.") + end + end + end + end + + # Reload local_preferences and mk_preferences as they may have been modified by MPI + # setup + local_preferences_filename = joinpath(repo_dir, "LocalPreferences.toml") + local_preferences = TOML.parsefile(local_preferences_filename) + mk_preferences = local_preferences["moment_kinetics"] + + mk_preferences["hdf5_dir"] = hdf5_dir + + # Delete any existing preferences for HDF5 and HDF5.jll because they may prevent + # `using HDF5` if the libraries do not exist. + pop!(local_preferences, "HDF5", nothing) + pop!(local_preferences, "HDF5_jll", nothing) + + open(local_preferences_filename, "w") do io + TOML.print(io, local_preferences, sorted=true) + end + + using HDF5 + if hdf5_dir == "default" + HDF5.API.set_libraries!() + else + HDF5.API.set_libraries!(hdf5_lib, hdf5_lib_hl) + end +else + error("Unrecognized setting " + * "hdf5_library_setting=$(machine_settings["hdf5_library_setting"])") +end diff --git a/machines/shared/machine_setup_stage_two.jl b/machines/shared/machine_setup_stage_two.jl index 3fdc3b091..a1f592e16 100644 --- a/machines/shared/machine_setup_stage_two.jl +++ b/machines/shared/machine_setup_stage_two.jl @@ -1,48 +1,10 @@ using Pkg, TOML -""" - get_input_with_path_completion(message=nothing) - -Print `message` and get user input using the `read` utility from bash to allow -path-completion. - -Solution adapted from -https://discourse.julialang.org/t/collecting-all-output-from-shell-commands/15592/2 -""" -function get_input_with_path_completion(message=nothing) - if message !== nothing - println(message) - end - - # The `out` object is used by `pipeline()` to capture output from the shell command. - out = Pipe() - - # Use Julia's shell command functionality to actually run bash - not sure how to use - # `read` and get output from it without using bash. - run(pipeline(`bash -c "read -e -p '> ' USERINPUT; echo \$USERINPUT"`, stdout=out)) - - # Need to close `out.in` to be able to read from `out`. - close(out.in) - - # chomp removes the trailing '\n'. 'String()' converts Vector{Char} to a String. - input = chomp(String(read(out))) - - return input -end - - repo_dir = dirname(dirname(dirname(@__FILE__))) -local_preferences = TOML.parsefile(joinpath(repo_dir, "LocalPreferences.toml")) +local_preferences_filename = joinpath(repo_dir, "LocalPreferences.toml") +local_preferences = TOML.parsefile(local_preferences_filename) mk_preferences = local_preferences["moment_kinetics"] -machine = mk_preferences["machine"] -machine_dir = joinpath(repo_dir, "machines", machine) -machine_settings_filename = joinpath(machine_dir, "machine_settings.toml") -if isfile(machine_settings_filename) - machine_settings = TOML.parsefile(machine_settings_filename) -else - machine_settings = Dict{String,Any}() -end batch_system = mk_preferences["batch_system"] @@ -67,141 +29,6 @@ if mk_preferences["use_plots"] == "y" end end -to_add = String["HDF5", "MPI", "MPIPreferences", "SpecialFunctions"] -to_rm = String[] -if mk_preferences["use_netcdf"] == "y" - push!(to_add, "NCDatasets") -else - push!(to_rm, "NCDatasets") -end -if mk_preferences["enable_mms"] == "y" - push!(to_add, "Symbolics", "IfElse") -else - push!(to_rm, "Symbolics", "IfElse") -end -for p ∈ to_rm - # If `p` was not previously installed, then Pkg.rm(p) with throw an error. We only - # need to remove if p was previously installed, so it is OK to ignore the error. - try - Pkg.rm(p) - catch - end -end -Pkg.add(to_add) - - -# Instantiate packages so we can use MPIPreferences below -######################################################### - -println("\n** Getting dependencies\n") -Pkg.instantiate() -Pkg.resolve() - - -# MPI setup -########### - -println("\n** Setting up to use system MPI\n") -using MPIPreferences - -if "mpi_library_names" ∈ keys(machine_settings) || "mpiexec" ∈ keys(machine_settings) - MPIPreferences.use_system_binary(library_names=machine_settings["mpi_library_names"], - mpiexec=machine_settings["mpiexec"]) -else - # If settings for MPI library are not given explicitly, then auto-detection by - # MPIPreferences.use_system_binary() should work. - MPIPreferences.use_system_binary() -end - - -# HDF5 setup -############ - -println("\n** Setting up to use system HDF5\n") - -if machine_settings["hdf5_library_setting"] == "system" - hdf5_dir = ENV["HDF5_DIR"] # system hdf5 - using HDF5 - HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), - joinpath(hdf5_dir, "libhdf5_hl.so")) -elseif machine_settings["hdf5_library_setting"] == "download" - artifact_dir = joinpath(repo_dir, "machines", "artifacts") - hdf5_dir = joinpath(artifact_dir, "hdf5-build/") - using HDF5 - HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), - joinpath(hdf5_dir, "libhdf5_hl.so")) -elseif machine_settings["hdf5_library_setting"] == "prompt" - # Prompt user to select what HDF5 to use - local_hdf5_install_dir = joinpath("machines", "artifacts", "hdf5-build", "lib") - if isdir(local_hdf5_install_dir) - local_hdf5_install_dir = realpath(local_hdf5_install_dir) - # We have downloaded and compiled HDF5, so link that - hdf5_dir = local_hdf5_install_dir - hdf5_lib = joinpath(local_hdf5_install_dir, "libhdf5.so") - hdf5_lib_hl = joinpath(local_hdf5_install_dir, "libhdf5_hl.so") - else - println("\n** Setting up to use system HDF5\n") - - default_hdf5_dir = get(ENV, "HDF5_DIR", "") # try to find a path to a system hdf5, may not work on all systems - - default_hdf5_dir = get(mk_preferences, "hdf5_dir", default_hdf5_dir) - - hdf5_dir = "" - hdf5_lib = "" - hdf5_lib_hl = "" - while true - global hdf5_dir, hdf5_lib, hdf5_lib_hl - hdf5_dir = get_input_with_path_completion( - "\nAn HDF5 installation compiled with your system MPI is required to use\n" - * "parallel I/O. Enter the directory where the libhdf5.so and libhdf5_hl.so are\n" - * "located (enter 'default' to use the Julia-provided HDF5, which does not\n" - * "support parallel I/O): [$default_hdf5_dir]") - - if hdf5_dir == "" - hdf5_dir = default_hdf5_dir - end - - if hdf5_dir == "default" - break - end - - if isdir(hdf5_dir) - hdf5_dir = realpath(hdf5_dir) - end - hdf5_lib = joinpath(hdf5_dir, "libhdf5.so") - hdf5_lib_hl = joinpath(hdf5_dir, "libhdf5_hl.so") - if isfile(hdf5_lib) && isfile(hdf5_lib_hl) - break - else - # Remove trailing slash if it exists so that we can print a single trailing slash - # consistently - hdf5_dir = rstrip(hdf5_dir, '/') - print("HDF5 libraries not found in '$hdf5_dir/'.") - if !isfile(hdf5_lib) - print(" $hdf5_lib does not exist.") - end - if !isfile(hdf5_lib_hl) - print(" $hdf5_lib_hl does not exist.") - end - end - end - - mk_preferences["hdf5_dir"] = hdf5_dir - open(local_preferences_filename, "w") do io - TOML.print(io, local_preferences, sorted=true) - end - end - using HDF5 - if hdf5_dir == "default" - HDF5.API.set_libraries!() - else - HDF5.API.set_libraries!(hdf5_lib, hdf5_lib_hl) - end -else - error("Unrecognized setting " - * "hdf5_library_setting=$(machine_settings["hdf5_library_setting"])") -end - Pkg.develop(path="moment_kinetics") Pkg.precompile() diff --git a/machines/shared/makie_post_processing_setup.jl b/machines/shared/makie_post_processing_setup.jl new file mode 100644 index 000000000..af11a190c --- /dev/null +++ b/machines/shared/makie_post_processing_setup.jl @@ -0,0 +1,46 @@ +using Pkg, TOML + +repo_dir = dirname(dirname(dirname(@__FILE__))) +local_preferences_filename = joinpath(repo_dir, "LocalPreferences.toml") +local_preferences = TOML.parsefile(local_preferences_filename) +mk_preferences = local_preferences["moment_kinetics"] +batch_system = mk_preferences["batch_system"] + +if mk_preferences["use_makie"] == "y" + println("Setting up makie_post_processing") + + if batch_system + touch(joinpath("makie_post_processing", "Project.toml")) + Pkg.activate("makie_post_processing") + + include("add_dependencies_to_project.jl") + Pkg.develop(path="moment_kinetics") + Pkg.add(["Makie", "CairoMakie"]) + Pkg.develop(path=joinpath("makie_post_processing", "makie_post_processing")) + Pkg.precompile() + + if mk_preferences["submit_precompilation"] == "y" + run(`precompile-makie-post-processing-submit.sh`) + end + else + Pkg.add(["Makie", "CairoMakie"]) + Pkg.develop(path=joinpath("makie_post_processing", "makie_post_processing")) + end +else + if !batch_system + # If makie_post_processing and dependencies have been added previously, remove + # them + try + Pkg.rm("makie_post_processing") + catch + end + try + Pkg.rm("Makie") + catch + end + try + Pkg.rm("CairoMakie") + catch + end + end +end diff --git a/machines/shared/plots_post_processing_setup.jl b/machines/shared/plots_post_processing_setup.jl new file mode 100644 index 000000000..3a7b39426 --- /dev/null +++ b/machines/shared/plots_post_processing_setup.jl @@ -0,0 +1,42 @@ +using Pkg, TOML + +repo_dir = dirname(dirname(dirname(@__FILE__))) +local_preferences_filename = joinpath(repo_dir, "LocalPreferences.toml") +local_preferences = TOML.parsefile(local_preferences_filename) +mk_preferences = local_preferences["moment_kinetics"] +batch_system = mk_preferences["batch_system"] + +if mk_preferences["use_plots"] == "y" + println("Setting up plots_post_processing") + + if batch_system + touch(joinpath("plots_post_processing", "Project.toml")) + Pkg.activate("plots_post_processing") + + include("add_dependencies_to_project.jl") + Pkg.develop(path="moment_kinetics") + Pkg.add("Plots") + Pkg.develop(path=joinpath("plots_post_processing", "plots_post_processing")) + Pkg.precompile() + + if mk_preferences["submit_precompilation"] == "y" + run(`precompile-plots-post-processing-submit.sh`) + end + else + Pkg.add("Plots") + Pkg.develop(path=joinpath("plots_post_processing", "plots_post_processing")) + end +else + if !batch_system + # If plots_post_processing and dependencies have been added previously, remove + # them + try + Pkg.rm("plots_post_processing") + catch + end + try + Pkg.rm("Plots") + catch + end + end +end From d6bda5722b1494adb6f47f30cd6a74082adecded Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 30 Dec 2023 22:01:23 +0000 Subject: [PATCH 37/80] Add optional dependencies after moment_kinetics, and switch order The order of precompiling the extensions `manufactured_solns_ext` and `file_io_netcdf` seems to be important - if `file_io_netcdf` is first, and error occurs during precompilation. Don't understand why! --- .../shared/add_dependencies_to_project.jl | 28 +++++++++++++------ machines/shared/machine_setup_stage_two.jl | 4 --- .../shared/makie_post_processing_setup.jl | 1 - .../shared/plots_post_processing_setup.jl | 1 - 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/machines/shared/add_dependencies_to_project.jl b/machines/shared/add_dependencies_to_project.jl index 3b9cae826..342c865d2 100644 --- a/machines/shared/add_dependencies_to_project.jl +++ b/machines/shared/add_dependencies_to_project.jl @@ -51,16 +51,11 @@ function get_input_with_path_completion(message=nothing) end -to_add = String["HDF5", "MPI", "MPIPreferences", "SpecialFunctions"] to_rm = String[] -if mk_preferences["use_netcdf"] == "y" - push!(to_add, "NCDatasets") -else +if mk_preferences["use_netcdf"] == "n" push!(to_rm, "NCDatasets") end -if mk_preferences["enable_mms"] == "y" - push!(to_add, "Symbolics", "IfElse") -else +if mk_preferences["enable_mms"] == "n" push!(to_rm, "Symbolics", "IfElse") end for p ∈ to_rm @@ -71,7 +66,7 @@ for p ∈ to_rm catch end end -Pkg.add(to_add) +Pkg.add(["HDF5", "MPI", "MPIPreferences", "SpecialFunctions"]) # Instantiate packages so we can use MPIPreferences below @@ -203,3 +198,20 @@ else error("Unrecognized setting " * "hdf5_library_setting=$(machine_settings["hdf5_library_setting"])") end + + +Pkg.develop(path="moment_kinetics") +Pkg.precompile() + + +# It seems to be important to add the dependencies for MMS before the ones for NetCDF (as +# of 30/12/2023). Don't understand why that should be true. +if mk_preferences["enable_mms"] == "y" + Pkg.add(["Symbolics", "IfElse"]) + Pkg.precompile() +end + +if mk_preferences["use_netcdf"] == "y" + Pkg.add("NCDatasets") + Pkg.precompile() +end diff --git a/machines/shared/machine_setup_stage_two.jl b/machines/shared/machine_setup_stage_two.jl index a1f592e16..209106a33 100644 --- a/machines/shared/machine_setup_stage_two.jl +++ b/machines/shared/machine_setup_stage_two.jl @@ -30,10 +30,6 @@ if mk_preferences["use_plots"] == "y" end -Pkg.develop(path="moment_kinetics") -Pkg.precompile() - - if batch_system # Make symlinks to batch job submission scripts symlink("precompile-submit.sh", joinpath("machines", "shared", "precompile-submit.sh")) diff --git a/machines/shared/makie_post_processing_setup.jl b/machines/shared/makie_post_processing_setup.jl index af11a190c..6d79372d5 100644 --- a/machines/shared/makie_post_processing_setup.jl +++ b/machines/shared/makie_post_processing_setup.jl @@ -14,7 +14,6 @@ if mk_preferences["use_makie"] == "y" Pkg.activate("makie_post_processing") include("add_dependencies_to_project.jl") - Pkg.develop(path="moment_kinetics") Pkg.add(["Makie", "CairoMakie"]) Pkg.develop(path=joinpath("makie_post_processing", "makie_post_processing")) Pkg.precompile() diff --git a/machines/shared/plots_post_processing_setup.jl b/machines/shared/plots_post_processing_setup.jl index 3a7b39426..1190ad92e 100644 --- a/machines/shared/plots_post_processing_setup.jl +++ b/machines/shared/plots_post_processing_setup.jl @@ -14,7 +14,6 @@ if mk_preferences["use_plots"] == "y" Pkg.activate("plots_post_processing") include("add_dependencies_to_project.jl") - Pkg.develop(path="moment_kinetics") Pkg.add("Plots") Pkg.develop(path=joinpath("plots_post_processing", "plots_post_processing")) Pkg.precompile() From cf64646fba3ad4ae3ab8b5c3c4cf211c49ce4702 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 31 Dec 2023 18:14:42 +0000 Subject: [PATCH 38/80] Add PackageCompiler to the top-level installed packages --- machines/shared/add_dependencies_to_project.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/shared/add_dependencies_to_project.jl b/machines/shared/add_dependencies_to_project.jl index 342c865d2..d0736fbdd 100644 --- a/machines/shared/add_dependencies_to_project.jl +++ b/machines/shared/add_dependencies_to_project.jl @@ -66,7 +66,7 @@ for p ∈ to_rm catch end end -Pkg.add(["HDF5", "MPI", "MPIPreferences", "SpecialFunctions"]) +Pkg.add(["HDF5", "MPI", "MPIPreferences", "PackageCompiler", "SpecialFunctions"]) # Instantiate packages so we can use MPIPreferences below From 2feee0fbdee3f5e6615852e0c52e95165a619e00 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 30 Dec 2023 12:48:36 +0000 Subject: [PATCH 39/80] Don't delete compiled HDF5 when it is deselected If the user has previously downloaded and compiled the HDF5 library, but chooses not to use it, do not delete the library as it may be re-selected in future. --- machines/generic-pc/compile_dependencies.sh | 9 ++++++--- machines/marconi/compile_dependencies.sh | 5 +++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/machines/generic-pc/compile_dependencies.sh b/machines/generic-pc/compile_dependencies.sh index 1159083b6..ae5f5a33e 100755 --- a/machines/generic-pc/compile_dependencies.sh +++ b/machines/generic-pc/compile_dependencies.sh @@ -50,9 +50,7 @@ fi # Save current response for whether to download/build HDF5 as default ../../bin/julia ../shared/set_mk_preference.jl build_hdf5 $BUILDHDF5 -if [[ BUILDHDF5 -eq 1 && -d hdf5-build ]]; then - rm -r hdf5-build -elif [[ BUILDHDF5 -eq 0 && -d hdf5-build ]]; then +if [[ BUILDHDF5 -eq 0 && -d hdf5-build ]]; then echo "HDF5 appears to have been downloaded, compiled and installed already." echo "Do you want to download, compile and install again, overwriting the existing " echo "version? y/[n]" @@ -64,6 +62,11 @@ elif [[ BUILDHDF5 -eq 0 && -d hdf5-build ]]; then done if [[ -z $input || $input == "n" ]]; then BUILDHDF5=1 + else + # Remove the install directory if it exists already + if [[ -d hdf5-build ]]; then + rm -r hdf5-build + fi fi fi diff --git a/machines/marconi/compile_dependencies.sh b/machines/marconi/compile_dependencies.sh index f47fad0cf..1e11e7420 100755 --- a/machines/marconi/compile_dependencies.sh +++ b/machines/marconi/compile_dependencies.sh @@ -21,6 +21,11 @@ if [ -d hdf5-build ]; then done if [[ -z $input || $input == "n" ]]; then BUILDHDF5=1 + else + # Remove the install directory if it exists already + if [[ -d hdf5-build ]]; then + rm -r hdf5-build + fi fi fi From 978e147942c37c47b37de6eb457d2209fb82e06b Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sat, 30 Dec 2023 23:59:20 +0000 Subject: [PATCH 40/80] Change build_hdf5 values to y/n instead of 0/1 --- machines/generic-pc/compile_dependencies.sh | 18 +++++++++--------- machines/shared/add_dependencies_to_project.jl | 2 +- machines/shared/get_mk_preference.jl | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/machines/generic-pc/compile_dependencies.sh b/machines/generic-pc/compile_dependencies.sh index ae5f5a33e..5a919271e 100755 --- a/machines/generic-pc/compile_dependencies.sh +++ b/machines/generic-pc/compile_dependencies.sh @@ -13,9 +13,9 @@ ARTIFACT_DIR=$PWD ###### # Get default response for whether to download/build HDF5 -DEFAULT_BUILDHDF5=$(../../bin/julia ../shared/get_mk_preference.jl build_hdf5 0) +DEFAULT_BUILDHDF5=$(../../bin/julia ../shared/get_mk_preference.jl build_hdf5 "y") -if [[ $DEFAULT_BUILDHDF5 -eq 0 ]]; then +if [[ $DEFAULT_BUILDHDF5 == "y" ]]; then echo "Do you want to download, and compile a local version of HDF5 (if you do" echo "not do this, you will be given the option to choose an HDF5 library to" echo "link later)? [y]/n" @@ -26,9 +26,9 @@ if [[ $DEFAULT_BUILDHDF5 -eq 0 ]]; then read -p "> " input done if [[ -z $input || $input == "y" ]]; then - BUILDHDF5=0 + BUILDHDF5="y" else - BUILDHDF5=1 + BUILDHDF5="n" fi else echo "Do you want to download, and compile a local version of HDF5 (if you do" @@ -41,16 +41,16 @@ else read -p "> " input done if [[ -z $input || $input == "n" ]]; then - BUILDHDF5=1 + BUILDHDF5="y" else - BUILDHDF5=0 + BUILDHDF5="n" fi fi # Save current response for whether to download/build HDF5 as default ../../bin/julia ../shared/set_mk_preference.jl build_hdf5 $BUILDHDF5 -if [[ BUILDHDF5 -eq 0 && -d hdf5-build ]]; then +if [[ $BUILDHDF5 == "y" && -d hdf5-build ]]; then echo "HDF5 appears to have been downloaded, compiled and installed already." echo "Do you want to download, compile and install again, overwriting the existing " echo "version? y/[n]" @@ -61,7 +61,7 @@ if [[ BUILDHDF5 -eq 0 && -d hdf5-build ]]; then read -p "> " input done if [[ -z $input || $input == "n" ]]; then - BUILDHDF5=1 + BUILDHDF5="n" else # Remove the install directory if it exists already if [[ -d hdf5-build ]]; then @@ -70,7 +70,7 @@ if [[ BUILDHDF5 -eq 0 && -d hdf5-build ]]; then fi fi -if [ $BUILDHDF5 -eq 0 ]; then +if [[ $BUILDHDF5 == "y" ]]; then HDF5=hdf5-1.14.3 # Download and extract the source wget -O ${HDF5}.tar.bz2 https://www.hdfgroup.org/package/hdf5-1-14-3-tar-bz2/?wpdmdl=18469 diff --git a/machines/shared/add_dependencies_to_project.jl b/machines/shared/add_dependencies_to_project.jl index d0736fbdd..350ba46d5 100644 --- a/machines/shared/add_dependencies_to_project.jl +++ b/machines/shared/add_dependencies_to_project.jl @@ -110,7 +110,7 @@ elseif machine_settings["hdf5_library_setting"] == "download" joinpath(hdf5_dir, "libhdf5_hl.so")) elseif machine_settings["hdf5_library_setting"] == "prompt" # Prompt user to select what HDF5 to use - if mk_preferences["build_hdf5"] == "0" + if mk_preferences["build_hdf5"] == "y" local_hdf5_install_dir = joinpath("machines", "artifacts", "hdf5-build", "lib") local_hdf5_install_dir = realpath(local_hdf5_install_dir) # We have downloaded and compiled HDF5, so link that diff --git a/machines/shared/get_mk_preference.jl b/machines/shared/get_mk_preference.jl index ae0acdfa6..a0ca8cfd4 100644 --- a/machines/shared/get_mk_preference.jl +++ b/machines/shared/get_mk_preference.jl @@ -6,7 +6,7 @@ preference_name = ARGS[1] if length(ARGS) > 1 default = ARGS[2] else - default = "1" + default = "n" end top_level_directory = dirname(dirname(dirname(@__FILE__))) From 7b17b6f4a6145d1f5d87703d14c3a9758f43c882 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 31 Dec 2023 00:07:09 +0000 Subject: [PATCH 41/80] Option to set up postprocessing in separate project on non-batch system --- machines/machine_setup.sh | 27 ++++++++++--------- machines/shared/machine_setup.jl | 8 ++++++ .../shared/makie_post_processing_setup.jl | 4 +-- .../shared/plots_post_processing_setup.jl | 4 +-- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 277f3b8bb..3c14668f2 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -179,19 +179,6 @@ else BATCH_SYSTEM=0 fi -if [[ $BATCH_SYSTEM -eq 0 ]]; then - # Batch systems can (conveniently) use different optimization flags for - # running simulations and for post-processing. - OPTIMIZATION_FLAGS="-O3 --check-bounds=no" - POSTPROC_OPTIMIZATION_FLAGS="-O3" -else - # On interactive systems which use the same project for running simulations - # and for post-processing, both should use the same optimization flags to - # avoid invalidating precompiled dependencies. - OPTIMIZATION_FLAGS="-O3" - POSTPROC_OPTIMIZATION_FLAGS=$OPTIMIZATION_FLAGS -fi - # Get the location for the .julia directory, in case this has to have a # non-default value, e.g. because the user's home directory is not accessible # from compute nodes. @@ -246,6 +233,20 @@ if [ -f julia.env ]; then source julia.env fi +SEPARATE_POSTPROC_PROJECTS=$(bin/julia machines/shared/get_mk_preference.jl separate_postproc_projects) +if [[ $BATCH_SYSTEM -eq 0 || $SEPARATE_POSTPROC_PROJECTS == "y" ]]; then + # Batch systems can (conveniently) use different optimization flags for + # running simulations and for post-processing. + OPTIMIZATION_FLAGS="-O3 --check-bounds=no" + POSTPROC_OPTIMIZATION_FLAGS="-O3" +else + # On interactive systems which use the same project for running simulations + # and for post-processing, both should use the same optimization flags to + # avoid invalidating precompiled dependencies. + OPTIMIZATION_FLAGS="-O3" + POSTPROC_OPTIMIZATION_FLAGS=$OPTIMIZATION_FLAGS +fi + # [ -f ] tests if exists and is a file if [ -f machines/shared/compile_dependencies.sh ]; then # Need to compile some dependencies diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index a436eeae1..168577d55 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -20,6 +20,7 @@ default_settings["base"] = Dict("account"=>"", "submit_precompilation"=>"y", "use_makie"=>"n", "use_plots"=>"n", + "separate_postproc_projects"=>"n", "use_netcdf"=>"n", "enable_mms"=>"n") # No batch system steup for "generic-pc" @@ -204,6 +205,13 @@ function machine_setup_moment_kinetics(machine::String; no_force_exit::Bool=fals get_setting("use_plots", "Would you like to set up plots_post_processing?", machine, mk_preferences, ["y", "n"]) + if !batch_system + get_setting("separate_postproc_projects", + "Would you like to set up separate packages for post processing (this might\n" + * "be useful if you want to use different optimization flags for runs and\n" + * "post-processing for example)?", + machine, mk_preferences, ["y", "n"]) + end get_setting("use_netcdf", "Would you like to enable optional NetCDF I/O (warning: using NetCDF sometimes\n" * "causes errors when using a local or system install of HDF5)?", diff --git a/machines/shared/makie_post_processing_setup.jl b/machines/shared/makie_post_processing_setup.jl index 6d79372d5..6c0d99e41 100644 --- a/machines/shared/makie_post_processing_setup.jl +++ b/machines/shared/makie_post_processing_setup.jl @@ -9,7 +9,7 @@ batch_system = mk_preferences["batch_system"] if mk_preferences["use_makie"] == "y" println("Setting up makie_post_processing") - if batch_system + if batch_system || mk_preferences["separate_postproc_projects"] == "y" touch(joinpath("makie_post_processing", "Project.toml")) Pkg.activate("makie_post_processing") @@ -18,7 +18,7 @@ if mk_preferences["use_makie"] == "y" Pkg.develop(path=joinpath("makie_post_processing", "makie_post_processing")) Pkg.precompile() - if mk_preferences["submit_precompilation"] == "y" + if batch_system && mk_preferences["submit_precompilation"] == "y" run(`precompile-makie-post-processing-submit.sh`) end else diff --git a/machines/shared/plots_post_processing_setup.jl b/machines/shared/plots_post_processing_setup.jl index 1190ad92e..7d2e297f7 100644 --- a/machines/shared/plots_post_processing_setup.jl +++ b/machines/shared/plots_post_processing_setup.jl @@ -9,7 +9,7 @@ batch_system = mk_preferences["batch_system"] if mk_preferences["use_plots"] == "y" println("Setting up plots_post_processing") - if batch_system + if batch_system || mk_preferences["separate_postproc_projects"] == "y" touch(joinpath("plots_post_processing", "Project.toml")) Pkg.activate("plots_post_processing") @@ -18,7 +18,7 @@ if mk_preferences["use_plots"] == "y" Pkg.develop(path=joinpath("plots_post_processing", "plots_post_processing")) Pkg.precompile() - if mk_preferences["submit_precompilation"] == "y" + if batch_system && mk_preferences["submit_precompilation"] == "y" run(`precompile-plots-post-processing-submit.sh`) end else From 96a776dc2f043a726f958e4d67469cd3e579a602 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 31 Dec 2023 14:35:14 +0000 Subject: [PATCH 42/80] Use relative path instead of symlink for shared_utils.jl Using a symlink seems to mean that Revise.jl fails to track changes in the shared file (although only when the file is also included somewhere without the symlink - it does track changes correctly if only the symlink is used). --- .../plots_post_processing/src/plots_post_processing.jl | 2 +- plots_post_processing/plots_post_processing/src/shared_utils.jl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 120000 plots_post_processing/plots_post_processing/src/shared_utils.jl diff --git a/plots_post_processing/plots_post_processing/src/plots_post_processing.jl b/plots_post_processing/plots_post_processing/src/plots_post_processing.jl index a6a7b1667..7303979d6 100644 --- a/plots_post_processing/plots_post_processing/src/plots_post_processing.jl +++ b/plots_post_processing/plots_post_processing/src/plots_post_processing.jl @@ -15,7 +15,7 @@ export get_coords_nelement export get_coords_ngrid include("post_processing_input.jl") -include("shared_utils.jl") +include("../../../makie_post_processing/makie_post_processing/src/shared_utils.jl") # Next three lines only used for workaround needed by plot_unnormalised() using PyCall diff --git a/plots_post_processing/plots_post_processing/src/shared_utils.jl b/plots_post_processing/plots_post_processing/src/shared_utils.jl deleted file mode 120000 index abdfe611b..000000000 --- a/plots_post_processing/plots_post_processing/src/shared_utils.jl +++ /dev/null @@ -1 +0,0 @@ -../../../makie_post_processing/makie_post_processing/src/shared_utils.jl \ No newline at end of file From 38e96fbca7a6a8218d16e6f0275b5e17e73449b0 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 31 Dec 2023 15:00:37 +0000 Subject: [PATCH 43/80] Print useful info showing optimization flags to use --- machines/machine_setup.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 3c14668f2..5383ebd69 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -267,5 +267,32 @@ echo "Finished!" if [[ $BATCH_SYSTEM -eq 0 ]]; then echo "Now run \`source julia.env\` to set up your environment, and/or add it to your .bashrc" fi +echo +if [[ $BATCH_SYSTEM -eq 0 ]]; then + echo "To run simulations interactively, start julia like:" + echo "$ bin/julia --project $OPTIMIZATION_FLAGS" + echo + echo "To run post-processing interactively, start julia like:" + echo "$ bin/julia --project=makie_post_processing $POSTPROC_OPTIMIZATION_FLAGS" + echo "or" + echo "$ bin/julia --project=plots_post_processing $POSTPROC_OPTIMIZATION_FLAGS" + echo + echo "Note that if you change the optimization flags '$OPTIMIZATION_FLAGS' or '$POSTPROC_OPTIMIZATION_FLAGS' precompilation may need to be repeated using the new flags (which is slow)." +elif [[ $SEPARATE_POSTPROC_PROJECTS == "y" ]]; then + echo "To run simulations, start julia like:" + echo "$ bin/julia --project $OPTIMIZATION_FLAGS" + echo + echo "To run post-processing, start julia like:" + echo "$ bin/julia --project=makie_post_processing $POSTPROC_OPTIMIZATION_FLAGS" + echo "or" + echo "$ bin/julia --project=plots_post_processing $POSTPROC_OPTIMIZATION_FLAGS" + echo + echo "Note that if you change the optimization flags '$OPTIMIZATION_FLAGS' or '$POSTPROC_OPTIMIZATION_FLAGS' precompilation may need to be repeated using the new flags (which is slow)." +else + echo "To run simulations or do post-processing, start julia like:" + echo "$ bin/julia --project $OPTIMIZATION_FLAGS" + echo + echo "Note that if you change the optimization flags '$OPTIMIZATION_FLAGS' precompilation may need to be repeated using the new flags (which is slow)." +fi exit 0 From 0a7db8808309fd26cb4913a92574f1612cb02509 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 31 Dec 2023 15:22:58 +0000 Subject: [PATCH 44/80] Uninstall postproc from main project when using separate postproc --- .../shared/makie_post_processing_setup.jl | 34 ++++++++++--------- .../shared/plots_post_processing_setup.jl | 25 +++++++------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/machines/shared/makie_post_processing_setup.jl b/machines/shared/makie_post_processing_setup.jl index 6c0d99e41..45d9b2a1c 100644 --- a/machines/shared/makie_post_processing_setup.jl +++ b/machines/shared/makie_post_processing_setup.jl @@ -25,21 +25,23 @@ if mk_preferences["use_makie"] == "y" Pkg.add(["Makie", "CairoMakie"]) Pkg.develop(path=joinpath("makie_post_processing", "makie_post_processing")) end -else - if !batch_system - # If makie_post_processing and dependencies have been added previously, remove - # them - try - Pkg.rm("makie_post_processing") - catch - end - try - Pkg.rm("Makie") - catch - end - try - Pkg.rm("CairoMakie") - catch - end +end + +if !batch_system && (mk_preferences["use_makie"] == "n" || + mk_preferences["separate_postproc_projects"] == "y") + # If makie_post_processing and dependencies have been added previously, remove + # them + Pkg.activate(".") + try + Pkg.rm("makie_post_processing") + catch + end + try + Pkg.rm("Makie") + catch + end + try + Pkg.rm("CairoMakie") + catch end end diff --git a/machines/shared/plots_post_processing_setup.jl b/machines/shared/plots_post_processing_setup.jl index 7d2e297f7..429047f4a 100644 --- a/machines/shared/plots_post_processing_setup.jl +++ b/machines/shared/plots_post_processing_setup.jl @@ -25,17 +25,18 @@ if mk_preferences["use_plots"] == "y" Pkg.add("Plots") Pkg.develop(path=joinpath("plots_post_processing", "plots_post_processing")) end -else - if !batch_system - # If plots_post_processing and dependencies have been added previously, remove - # them - try - Pkg.rm("plots_post_processing") - catch - end - try - Pkg.rm("Plots") - catch - end +end +if !batch_system && (mk_preferences["use_plots"] == "n" || + mk_preferences["separate_postproc_projects"] == "y") + # If plots_post_processing and dependencies have been added previously, remove + # them + Pkg.activate(".") + try + Pkg.rm("plots_post_processing") + catch + end + try + Pkg.rm("Plots") + catch end end From 8e26a96bc87f82e2e761f193d8ede71b4e824e54 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 31 Dec 2023 16:38:58 +0000 Subject: [PATCH 45/80] Add 'generic-batch' target in setup scripts 'generic-batch' can be used (with a bit of manual editing of settings/template files) for a generic batch HPC system. --- .gitignore | 7 +- machines/generic-batch-template/README.md | 26 ++++++ .../compile_dependencies.sh | 93 +++++++++++++++++++ machines/generic-batch-template/get-julia.sh | 8 ++ .../jobscript-postprocess-plotsjl.template | 19 ++++ .../jobscript-postprocess.template | 19 ++++ ...-precompile-makie-post-processing.template | 20 ++++ .../jobscript-precompile-no-run.template | 20 ++++ ...-precompile-plots-post-processing.template | 20 ++++ .../jobscript-precompile.template | 20 ++++ .../jobscript-restart.template | 24 +++++ .../jobscript-run.template | 24 +++++ machines/generic-batch-template/julia.env | 16 ++++ .../machine_settings.toml | 19 ++++ machines/machine_setup.sh | 24 ++++- machines/shared/machine_setup.jl | 12 +-- 16 files changed, 358 insertions(+), 13 deletions(-) create mode 100644 machines/generic-batch-template/README.md create mode 100755 machines/generic-batch-template/compile_dependencies.sh create mode 100755 machines/generic-batch-template/get-julia.sh create mode 100644 machines/generic-batch-template/jobscript-postprocess-plotsjl.template create mode 100644 machines/generic-batch-template/jobscript-postprocess.template create mode 100644 machines/generic-batch-template/jobscript-precompile-makie-post-processing.template create mode 100644 machines/generic-batch-template/jobscript-precompile-no-run.template create mode 100644 machines/generic-batch-template/jobscript-precompile-plots-post-processing.template create mode 100644 machines/generic-batch-template/jobscript-precompile.template create mode 100644 machines/generic-batch-template/jobscript-restart.template create mode 100644 machines/generic-batch-template/jobscript-run.template create mode 100644 machines/generic-batch-template/julia.env create mode 100644 machines/generic-batch-template/machine_settings.toml diff --git a/.gitignore b/.gitignore index 4de5cd7f4..5c2c8fdf2 100644 --- a/.gitignore +++ b/.gitignore @@ -7,9 +7,10 @@ LocalPreferences.toml /plots_post_processing/Project.toml /julia.env /.julia -bin -machines/shared/compile_dependencies.sh -machines/artifacts +/bin +/machines/shared/compile_dependencies.sh +/machines/artifacts +/machines/generic-batch /precompile-makie-post-processing-submit.sh /precompile-plots-post-processing-submit.sh /precompile-submit.sh diff --git a/machines/generic-batch-template/README.md b/machines/generic-batch-template/README.md new file mode 100644 index 000000000..65ec8005e --- /dev/null +++ b/machines/generic-batch-template/README.md @@ -0,0 +1,26 @@ +To use `generic-batch` you must copy `machines/generic-batch-template` to +`machines/generic-batch` and: + +* Edit the modules in `machines/generic-batch/julia.env` (see comments in that + file) +* Edit the `jobscript-*.template` files for precompilation or post-processing + jobs with the correct serial or debug queue for your machine. +* If you want to use a system-provided HDF5 you can delete + `machines/generic-batch/compile_dependencies.sh`, and uncomment the + `hdf5_library_setting = "system"` option in + `machines/generic-batch/machine_settings.toml` +* If `MPIPreferences.use_system_binary()` cannot auto-detect your MPI library, + then you may need to set the `mpi_library_names` setting in + `machines/generic-batch/machine_settings.toml` +* If `MPIPreferences.use_system_binary()` cannot auto-detect your MPI library + and/or if `mpirun` is not the right command to launch MPI processes, then you + need to set the `mpi_library_names` and `mpiexec` settings in + `machines/generic-batch/machine_settings.toml` (note if either of these + settings is set, then both must be) +* If `mpirun` is not the right command to launch MPI processes, you may need to + edit the `jobscript-run.template` and `jobscript-restart.template` files in + `machines/generic-batch/` and set the `mpiexec` setting in + `machines/generic-batch/machine_settings.toml` + +Note that `generic-batch` is set up assuming a Linux, x86\_64 based machine +that uses the 'module' system and a SLURM job queue. diff --git a/machines/generic-batch-template/compile_dependencies.sh b/machines/generic-batch-template/compile_dependencies.sh new file mode 100755 index 000000000..f4da5795a --- /dev/null +++ b/machines/generic-batch-template/compile_dependencies.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +# Download and compile dependencies +################################### +# +# Note that you can delete this script if you only want to use a +# system-provided HDF5 library. + +set -e + +if [ -f julia.env ]; then + # Set up modules, JULIA_DEPOT_PATH, etc. to use for the rest of this script + source julia.env +fi +cd machines/artifacts/ +ARTIFACT_DIR=$PWD + +# HDF5 +###### + +# Get default response for whether to download/build HDF5 +DEFAULT_BUILDHDF5=$(../../bin/julia ../shared/get_mk_preference.jl build_hdf5 "y") + +if [[ $DEFAULT_BUILDHDF5 == "y" ]]; then + echo "Do you want to download, and compile a local version of HDF5 (if you do" + echo "not do this, you will be given the option to choose an HDF5 library to" + echo "link later)? [y]/n" + read -p "> " input + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + done + if [[ -z $input || $input == "y" ]]; then + BUILDHDF5="y" + else + BUILDHDF5="n" + fi +else + echo "Do you want to download, and compile a local version of HDF5 (if you do" + echo "not do this, you will be given the option to choose an HDF5 library to" + echo "link later)? y/[n]" + read -p "> " input + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + done + if [[ -z $input || $input == "n" ]]; then + BUILDHDF5="y" + else + BUILDHDF5="n" + fi +fi + +# Save current response for whether to download/build HDF5 as default +../../bin/julia ../shared/set_mk_preference.jl build_hdf5 $BUILDHDF5 + +if [[ $BUILDHDF5 == "y" && -d hdf5-build ]]; then + echo "HDF5 appears to have been downloaded, compiled and installed already." + echo "Do you want to download, compile and install again, overwriting the existing " + echo "version? y/[n]" + read -p "> " input + while [[ ! -z $input && !( $input == "y" || $input == "n" ) ]]; do + echo + echo "$input is not a valid response: y/[n]" + read -p "> " input + done + if [[ -z $input || $input == "n" ]]; then + BUILDHDF5="n" + else + # Remove the install directory if it exists already + if [[ -d hdf5-build ]]; then + rm -r hdf5-build + fi + fi +fi + +if [[ $BUILDHDF5 == "y" ]]; then + HDF5=hdf5-1.14.3 + # Download and extract the source + wget -O ${HDF5}.tar.bz2 https://www.hdfgroup.org/package/hdf5-1-14-3-tar-bz2/?wpdmdl=18469 + tar xjf ${HDF5}.tar.bz2 + + cd $HDF5 + + # Configure and compile + CC=mpicc ./configure --enable-parallel --prefix=$ARTIFACT_DIR/hdf5-build/ + make -j 4 + make install +fi + +exit 0 diff --git a/machines/generic-batch-template/get-julia.sh b/machines/generic-batch-template/get-julia.sh new file mode 100755 index 000000000..625aee034 --- /dev/null +++ b/machines/generic-batch-template/get-julia.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Load modules to make sure we have a sensible Python +if [[ -f machines/generic-batch/julia.env ]]; then + source machines/generic-batch/julia.env +fi + +machines/shared/get-julia-linux-x86_64.sh $@ diff --git a/machines/generic-batch-template/jobscript-postprocess-plotsjl.template b/machines/generic-batch-template/jobscript-postprocess-plotsjl.template new file mode 100644 index 000000000..1bc71d243 --- /dev/null +++ b/machines/generic-batch-template/jobscript-postprocess-plotsjl.template @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +#SBATCH --ntasks=1 +#SBATCH --time=POSTPROCTIME +#SBATCH --account=ACCOUNT +#SBATCH --partition=some-serial-partition +#SBATCH --output=RUNDIRslurm-post-%j.out + +set -e + +cd $SLURM_SUBMIT_DIR + +# Get setup for Julia +source julia.env + +echo "post-processing (with original post_processing) RUNDIR $(date)" +bin/julia -Jplots_postproc.so --project=plots_post_processing/ run_post_processing.jl RUNDIR + +echo "finished post-processing RUNDIR $(date)" diff --git a/machines/generic-batch-template/jobscript-postprocess.template b/machines/generic-batch-template/jobscript-postprocess.template new file mode 100644 index 000000000..b21abac68 --- /dev/null +++ b/machines/generic-batch-template/jobscript-postprocess.template @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +#SBATCH --ntasks=1 +#SBATCH --time=POSTPROCTIME +#SBATCH --account=ACCOUNT +#SBATCH --partition=some-serial-partition +#SBATCH --output=RUNDIRslurm-post-%j.out + +set -e + +cd $SLURM_SUBMIT_DIR + +# Get setup for Julia +source julia.env + +echo "post-processing RUNDIR $(date)" +bin/julia -Jmakie_postproc.so --project=makie_post_processing/ run_makie_post_processing.jl RUNDIR + +echo "finished post-processing RUNDIR $(date)" diff --git a/machines/generic-batch-template/jobscript-precompile-makie-post-processing.template b/machines/generic-batch-template/jobscript-precompile-makie-post-processing.template new file mode 100644 index 000000000..8a0aefa5c --- /dev/null +++ b/machines/generic-batch-template/jobscript-precompile-makie-post-processing.template @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +#SBATCH --ntasks=1 +#SBATCH --time=1:00:00 +#SBATCH --account=ACCOUNT +#SBATCH --partition=some-serial-partition +#SBATCH --output=PRECOMPILEDIRslurm-%j.out + +set -e + +cd $SLURM_SUBMIT_DIR + +# Get setup for Julia +source julia.env + +echo "precompiling $(date)" + +bin/julia --project=makie_post_processing/ precompile-makie-post-processing.jl + +echo "finished!" diff --git a/machines/generic-batch-template/jobscript-precompile-no-run.template b/machines/generic-batch-template/jobscript-precompile-no-run.template new file mode 100644 index 000000000..43aabcb69 --- /dev/null +++ b/machines/generic-batch-template/jobscript-precompile-no-run.template @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +#SBATCH --ntasks=1 +#SBATCH --time=1:00:00 +#SBATCH --account=ACCOUNT +#SBATCH --partition=some-serial-partition +#SBATCH --output=PRECOMPILEDIRslurm-%j.out + +set -e + +cd $SLURM_SUBMIT_DIR + +# Get setup for Julia +source julia.env + +echo "precompiling $(date)" + +bin/julia --project -O3 --check-bounds=no precompile-no-run.jl + +echo "finished!" diff --git a/machines/generic-batch-template/jobscript-precompile-plots-post-processing.template b/machines/generic-batch-template/jobscript-precompile-plots-post-processing.template new file mode 100644 index 000000000..80d5736bd --- /dev/null +++ b/machines/generic-batch-template/jobscript-precompile-plots-post-processing.template @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +#SBATCH --ntasks=1 +#SBATCH --time=1:00:00 +#SBATCH --account=ACCOUNT +#SBATCH --partition=some-serial-partition +#SBATCH --output=PRECOMPILEDIRslurm-%j.out + +set -e + +cd $SLURM_SUBMIT_DIR + +# Get setup for Julia +source julia.env + +echo "precompiling $(date)" + +bin/julia --project=plots_post_processing/ precompile-plots-post-processing.jl + +echo "finished!" diff --git a/machines/generic-batch-template/jobscript-precompile.template b/machines/generic-batch-template/jobscript-precompile.template new file mode 100644 index 000000000..0d289fc5a --- /dev/null +++ b/machines/generic-batch-template/jobscript-precompile.template @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +#SBATCH --ntasks=1 +#SBATCH --time=1:00:00 +#SBATCH --account=ACCOUNT +#SBATCH --partition=some-serial-partition +#SBATCH --output=PRECOMPILEDIRslurm-%j.out + +set -e + +cd $SLURM_SUBMIT_DIR + +# Get setup for Julia +source julia.env + +echo "precompiling $(date)" + +bin/julia --project -O3 --check-bounds=no precompile.jl + +echo "finished!" diff --git a/machines/generic-batch-template/jobscript-restart.template b/machines/generic-batch-template/jobscript-restart.template new file mode 100644 index 000000000..52e70a63f --- /dev/null +++ b/machines/generic-batch-template/jobscript-restart.template @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +#SBATCH --nodes=NODES +#SBATCH --cpus-per-task=1 +#SBATCH --ntasks-per-node=48 +#SBATCH --time=RUNTIME +#SBATCH --account=ACCOUNT +#SBATCH --partition=PARTITION +#SBATCH --qos=QOS +#SBATCH --output=RUNDIRslurm-%j.out + +set -e + +cd $SLURM_SUBMIT_DIR + +# Get setup for Julia +source julia.env + +echo "running INPUTFILE $(date)" + +# May need to change this if mpirun` is not what should be used on your system +mpirun -np $SLURM_NTASKS bin/julia -Jmoment_kinetics.so --project -O3 --check-bounds=no run_moment_kinetics.jl --restart INPUTFILE RESTARTFROM + +echo "finished INPUTFILE $(date)" diff --git a/machines/generic-batch-template/jobscript-run.template b/machines/generic-batch-template/jobscript-run.template new file mode 100644 index 000000000..dce67e2c9 --- /dev/null +++ b/machines/generic-batch-template/jobscript-run.template @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +#SBATCH --nodes=NODES +#SBATCH --cpus-per-task=1 +#SBATCH --ntasks-per-node=48 +#SBATCH --time=RUNTIME +#SBATCH --account=ACCOUNT +#SBATCH --partition=PARTITION +#SBATCH --qos=QOS +#SBATCH --output=RUNDIRslurm-%j.out + +set -e + +cd $SLURM_SUBMIT_DIR + +# Get setup for Julia +source julia.env + +echo "running INPUTFILE $(date)" + +# May need to change this if mpirun` is not what should be used on your system +mpirun -np $SLURM_NTASKS bin/julia -Jmoment_kinetics.so --project -O3 --check-bounds=no run_moment_kinetics.jl INPUTFILE + +echo "finished INPUTFILE $(date)" diff --git a/machines/generic-batch-template/julia.env b/machines/generic-batch-template/julia.env new file mode 100644 index 000000000..5b25683f4 --- /dev/null +++ b/machines/generic-batch-template/julia.env @@ -0,0 +1,16 @@ +# Set up environment for julia and moment_kinetics + +module purge + +# Note the following needs to be edited with some sensible modules for your +# system. +# Modules to use system HDF5 (if you use this, you can delete +# compile_dependencies.sh): +module load some-mpi hdf5_some-mpi +# +# Or, if not using systm HDF5, load some compilers so that we can download and +# compile HDF5 locally: +#module load some-compiler some-mpi + +# Ensure Julia can find the system `python` correctly +PYTHON=python3 diff --git a/machines/generic-batch-template/machine_settings.toml b/machines/generic-batch-template/machine_settings.toml new file mode 100644 index 000000000..1dde887f3 --- /dev/null +++ b/machines/generic-batch-template/machine_settings.toml @@ -0,0 +1,19 @@ +# HDF5 settings +############### + +#hdf5_library_setting = "system" # Always use system HDF5 +#hdf5_library_setting = "download" # Always download HDF5 +hdf5_library_setting = "prompt" # Prompt for whether to download HDF5 when running machines/machine_setup.sh + + +# MPI settings +############## + +# You might need to set this if MPIPreferences.use_system_binary() cannot +# automatically find the MPI library +#mpi_library_names = "libmpi_something_00" + +# You might need to set this if `mpiexec` is not what should be used to run MPI +# programs (although this setting is probably actually not used, and only the +# commands in the jobscript templates are important). +#mpiexec = "srun" diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 5383ebd69..7695f155b 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -40,6 +40,8 @@ else DEFAULT_MACHINE=marconi elif module avail 2>&1 | grep -q epcc; then DEFAULT_MACHINE=archer + elif $(command -v module avail); then + DEFAULT_MACHINE=generic-batch else DEFAULT_MACHINE=generic-pc fi @@ -96,9 +98,9 @@ else fi # Get name of 'machine' -while [[ -z $MACHINE || !( $MACHINE == "generic-pc" || $MACHINE == "archer" || $MACHINE == "marconi" ) ]]; do +while [[ -z $MACHINE || !( $MACHINE == "generic-pc" || $MACHINE == "generic-batch" || $MACHINE == "archer" || $MACHINE == "marconi" ) ]]; do echo "Enter name of the machine to set up (must be one of 'generic-pc'," - echo "'archer', or 'marconi') [$DEFAULT_MACHINE]:" + echo "'generic-batch', 'archer', or 'marconi') [$DEFAULT_MACHINE]:" read -p "> " MACHINE echo if [ -z $MACHINE ]; then @@ -106,6 +108,24 @@ while [[ -z $MACHINE || !( $MACHINE == "generic-pc" || $MACHINE == "archer" || $ fi done +if [[ $MACHINE == "generic-batch" && ! -d machines/generic-batch ]]; then + echo "To use 'generic-batch' you must copy 'machines/generic-batch-template' to 'machines/generic-batch' and:" + echo "* Edit the modules in 'machines/generic-batch/julia.env' (see comments in that file)" + echo "* Edit the 'jobscript-*.template' files for precompilation or post-processing jobswith the correct serial or" + echo " debug queue for your machine." + echo "* If you want to use a system-provided HDF5 you can delete 'machines/generic-batch/compile_dependencies.sh'," + echo " and uncomment the 'hdf5_library_setting = \"system\"' option in 'machines/generic-batch/machine_settings.toml'" + echo "* If 'MPIPreferences.use_system_binary()' cannot auto-detect your MPI library and/or if 'mpirun' is not the " + echo " right command to launch MPI processes, then you need to set the 'mpi_library_names' and 'mpiexec' settings in" + echo " 'machines/generic-batch/machine_settings.toml' (note if either of these settings is set, then both must be)" + echo "* If 'mpirun' is not the right command to launch MPI processes, you may need to edit the 'jobscript-run.template'" + echo " and 'jobscript-restart.template' files in 'machines/generic-batch/' and set the setting in" + echo " 'machines/generic-batch/machine_settings.toml'" + echo "Note that 'generic-batch' is set up assuming a Linux, x86_64 based machine that uses the 'module' system and a" + echo "SLURM job queue." + exit 1 +fi + echo "Setting up for '$MACHINE'" echo diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index 168577d55..5dc696722 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -30,6 +30,7 @@ default_settings["generic-pc"] = merge(default_settings["base"], "default_postproc_time"=>"0:00:00", "default_postproc_memory"=>"0", "use_makie"=>"y")) +default_settings["generic-batch"] = deepcopy(default_settings["base"]) default_settings["archer"] = merge(default_settings["base"], Dict("default_partition"=>"standard", "default_qos"=>"standard")) @@ -292,16 +293,14 @@ function machine_setup_moment_kinetics(machine::String; no_force_exit::Bool=fals "compile_dependencies.sh") compile_dependencies_path = joinpath(repo_dir, compile_dependencies_relative_path) - needs_compile_dependencies = false # Set this flag to true in the machine-specific branch below to require a # non-empty `account` setting needs_account = false if machine == "generic-pc" - # For generic-pc, run compile_dependencies.sh script to optionally download and - # compile HDF5 - needs_compile_dependencies = true + elseif machine == "generic-batch" + needs_account = true elseif machine == "archer" needs_account = true if julia_directory == "" @@ -312,9 +311,6 @@ function machine_setup_moment_kinetics(machine::String; no_force_exit::Bool=fals end elseif machine == "marconi" needs_account = true - - # For marconi, need to run a script to compile HDF5 - needs_compile_dependencies = true else error("Unsupported machine '$machine'") end @@ -324,7 +320,7 @@ function machine_setup_moment_kinetics(machine::String; no_force_exit::Bool=fals * "`account` argument.") end - if needs_compile_dependencies + if isfile(joinpath("machines", machine, "compile_dependencies.sh")) # Remove link if it exists already islink(compile_dependencies_path) && rm(compile_dependencies_path) From 1501bc0bcaa53e96c94eade0331881318e3e6061 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 31 Dec 2023 16:55:09 +0000 Subject: [PATCH 46/80] Fix path for 'system' HDF5 --- machines/shared/add_dependencies_to_project.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/shared/add_dependencies_to_project.jl b/machines/shared/add_dependencies_to_project.jl index 350ba46d5..7c49807e5 100644 --- a/machines/shared/add_dependencies_to_project.jl +++ b/machines/shared/add_dependencies_to_project.jl @@ -98,7 +98,7 @@ end println("\n** Setting up to use system HDF5\n") if machine_settings["hdf5_library_setting"] == "system" - hdf5_dir = ENV["HDF5_DIR"] # system hdf5 + hdf5_dir = joinpath(ENV["HDF5_DIR"], "lib") # system hdf5 using HDF5 HDF5.API.set_libraries!(joinpath(hdf5_dir, "libhdf5.so"), joinpath(hdf5_dir, "libhdf5_hl.so")) From f7deca2423d0a6fe29b6edc972795b5cfbb7f537 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 31 Dec 2023 17:17:29 +0000 Subject: [PATCH 47/80] Fix creation of symlinks to batch submission scripts --- machines/shared/machine_setup_stage_two.jl | 26 ++++++++++--------- .../shared/makie_post_processing_setup.jl | 2 +- .../shared/plots_post_processing_setup.jl | 2 +- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/machines/shared/machine_setup_stage_two.jl b/machines/shared/machine_setup_stage_two.jl index 209106a33..a528b9835 100644 --- a/machines/shared/machine_setup_stage_two.jl +++ b/machines/shared/machine_setup_stage_two.jl @@ -32,22 +32,24 @@ end if batch_system # Make symlinks to batch job submission scripts - symlink("precompile-submit.sh", joinpath("machines", "shared", "precompile-submit.sh")) - symlink("submit-run.sh", joinpath("machines", "shared", "submit-run.sh")) - symlink("submit-restart.sh", joinpath("machines", "shared", "submit-restart.sh")) - if mk_preferences["use_makie"] - symlink("precompile-makie-post-processing-submit.sh", - joinpath("machines", "shared", - "precompile-makie-post-processing-submit.sh")) + function make_batch_symlink(script_name) + if islink(script_name) + rm(script_name) + end + symlink(joinpath("machines", "shared", script_name), script_name) end - if mk_preferences["use_plots"] - symlink("precompile-plots-post-processing-submit.sh", - joinpath("machines", "shared", - "precompile-plots-post-processing-submit.sh")) + make_batch_symlink("precompile-submit.sh") + make_batch_symlink("submit-run.sh") + make_batch_symlink("submit-restart.sh") + if mk_preferences["use_makie"] == "y" + make_batch_symlink("precompile-makie-post-processing-submit.sh") + end + if mk_preferences["use_plots"] == "y" + make_batch_symlink("precompile-plots-post-processing-submit.sh") end if mk_preferences["submit_precompilation"] == "y" - run(`precompile-submit.sh`) + run(`./precompile-submit.sh`) end end diff --git a/machines/shared/makie_post_processing_setup.jl b/machines/shared/makie_post_processing_setup.jl index 45d9b2a1c..e331471c4 100644 --- a/machines/shared/makie_post_processing_setup.jl +++ b/machines/shared/makie_post_processing_setup.jl @@ -19,7 +19,7 @@ if mk_preferences["use_makie"] == "y" Pkg.precompile() if batch_system && mk_preferences["submit_precompilation"] == "y" - run(`precompile-makie-post-processing-submit.sh`) + run(`./precompile-makie-post-processing-submit.sh`) end else Pkg.add(["Makie", "CairoMakie"]) diff --git a/machines/shared/plots_post_processing_setup.jl b/machines/shared/plots_post_processing_setup.jl index 429047f4a..f13002709 100644 --- a/machines/shared/plots_post_processing_setup.jl +++ b/machines/shared/plots_post_processing_setup.jl @@ -19,7 +19,7 @@ if mk_preferences["use_plots"] == "y" Pkg.precompile() if batch_system && mk_preferences["submit_precompilation"] == "y" - run(`precompile-plots-post-processing-submit.sh`) + run(`./precompile-plots-post-processing-submit.sh`) end else Pkg.add("Plots") From 24ea2864c51d093873efb71d02ffbd0624141ab2 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 31 Dec 2023 18:12:27 +0000 Subject: [PATCH 48/80] Remove Preferences dependence in get-precompile-info.jl, get-job-info.jl In the setup scripts we just use TOML to open LocalPreferences.toml directly (because Preferences requires the package to be loaded before loading/setting preferences for that package, but we want to save settings before loading moment_kinetics), so do the same in util/get-precompile-info.jl and util/get-job-info.jl rather than adding the Preferences package to the top-level environment. --- util/get-job-info.jl | 30 ++++++++++++++++++++---------- util/get-precompile-info.jl | 29 +++++++++++++++++++---------- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/util/get-job-info.jl b/util/get-job-info.jl index 12a73eff4..12fa1a128 100755 --- a/util/get-job-info.jl +++ b/util/get-job-info.jl @@ -8,22 +8,32 @@ Get information needed for submitting a run """ -using Preferences -using UUIDs +using TOML + + +function failure() + println("Missing `$name` preference. Need to run `setup_moment_kinetics()` before " + * "using this script.") + exit(1) +end settings_string = "" -# Use the UUID to get preferences to avoid having to import moment_kinetics here, which -# takes a while -moment_kinetics_uuid = UUID("b5ff72cc-06fc-4161-ad14-dba1c22ed34e") +if !isfile("LocalPreferences.toml") + failure() +end +local_preferences = TOML.parsefile("LocalPreferences.toml") + +if "moment_kinetics" ∉ keys(local_preferences) + failure() +end +mk_preferences = local_preferences["moment_kinetics"] function check_and_get_pref(name) - if !has_preference(moment_kinetics_uuid, name) - println("Missing `$name` preference. Need to run `setup_moment_kinetics()` before " - * "using this script.") - exit(1) + if name ∉ keys(mk_preferences) + failure() end - return string(load_preference(moment_kinetics_uuid, name)) * " " + return string(mk_preferences[name]) * " " end settings_string *= check_and_get_pref("machine") settings_string *= check_and_get_pref("account") diff --git a/util/get-precompile-info.jl b/util/get-precompile-info.jl index 7ba7de824..0264a2400 100755 --- a/util/get-precompile-info.jl +++ b/util/get-precompile-info.jl @@ -8,22 +8,31 @@ Get information needed for precompiling """ -using Preferences -using UUIDs +using TOML -# Use the UUID to get preferences to avoid having to import moment_kinetics here, which -# takes a while -moment_kinetics_uuid = UUID("b5ff72cc-06fc-4161-ad14-dba1c22ed34e") +function failure() + println("Missing `$name` preference. Need to run `setup_moment_kinetics()` before " + * "using this script.") + exit(1) +end settings_string = "" +if !isfile("LocalPreferences.toml") + failure() +end +local_preferences = TOML.parsefile("LocalPreferences.toml") + +if "moment_kinetics" ∉ keys(local_preferences) + failure() +end +mk_preferences = local_preferences["moment_kinetics"] + function check_and_get_pref(name) - if !has_preference(moment_kinetics_uuid, name) - println("Missing `$name` preference. Need to run `setup_moment_kinetics()` before " - * "using this script.") - exit(1) + if name ∉ keys(mk_preferences) + failure() end - return string(load_preference(moment_kinetics_uuid, name)) * " " + return string(mk_preferences[name]) * " " end settings_string *= check_and_get_pref("machine") settings_string *= check_and_get_pref("account") From 6dc89280427d63f0668d20dd7b224c7c4c8e20ab Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 31 Dec 2023 18:43:27 +0000 Subject: [PATCH 49/80] Better way of getting repo_dir ...that works also when setting up separate projects in the makie_post_processing and/or plots_post_processing subdirectories --- machines/shared/add_dependencies_to_project.jl | 2 +- machines/shared/machine_setup_stage_two.jl | 2 +- machines/shared/makie_post_processing_setup.jl | 2 +- machines/shared/plots_post_processing_setup.jl | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/machines/shared/add_dependencies_to_project.jl b/machines/shared/add_dependencies_to_project.jl index 7c49807e5..e386fa70d 100644 --- a/machines/shared/add_dependencies_to_project.jl +++ b/machines/shared/add_dependencies_to_project.jl @@ -2,7 +2,7 @@ using Pkg, TOML if abspath(PROGRAM_FILE) == @__FILE__ prompt_for_hdf5 = true - repo_dir = dirname(dirname(dirname(@__FILE__))) + repo_dir = dirname(Pkg.project().path) local_preferences_filename = joinpath(repo_dir, "LocalPreferences.toml") local_preferences = TOML.parsefile(local_preferences_filename) mk_preferences = local_preferences["moment_kinetics"] diff --git a/machines/shared/machine_setup_stage_two.jl b/machines/shared/machine_setup_stage_two.jl index a528b9835..45b141a0e 100644 --- a/machines/shared/machine_setup_stage_two.jl +++ b/machines/shared/machine_setup_stage_two.jl @@ -1,7 +1,7 @@ using Pkg, TOML -repo_dir = dirname(dirname(dirname(@__FILE__))) +repo_dir = dirname(Pkg.project().path) local_preferences_filename = joinpath(repo_dir, "LocalPreferences.toml") local_preferences = TOML.parsefile(local_preferences_filename) mk_preferences = local_preferences["moment_kinetics"] diff --git a/machines/shared/makie_post_processing_setup.jl b/machines/shared/makie_post_processing_setup.jl index e331471c4..6cf16054f 100644 --- a/machines/shared/makie_post_processing_setup.jl +++ b/machines/shared/makie_post_processing_setup.jl @@ -1,6 +1,6 @@ using Pkg, TOML -repo_dir = dirname(dirname(dirname(@__FILE__))) +repo_dir = dirname(Pkg.project().path) local_preferences_filename = joinpath(repo_dir, "LocalPreferences.toml") local_preferences = TOML.parsefile(local_preferences_filename) mk_preferences = local_preferences["moment_kinetics"] diff --git a/machines/shared/plots_post_processing_setup.jl b/machines/shared/plots_post_processing_setup.jl index f13002709..559ed28b8 100644 --- a/machines/shared/plots_post_processing_setup.jl +++ b/machines/shared/plots_post_processing_setup.jl @@ -1,6 +1,6 @@ using Pkg, TOML -repo_dir = dirname(dirname(dirname(@__FILE__))) +repo_dir = dirname(Pkg.project().path) local_preferences_filename = joinpath(repo_dir, "LocalPreferences.toml") local_preferences = TOML.parsefile(local_preferences_filename) mk_preferences = local_preferences["moment_kinetics"] From 7495eabb0a7e86203d56236418a0d432c40a804e Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 31 Dec 2023 19:10:48 +0000 Subject: [PATCH 50/80] Fix system-image compilation of post-processing packages --- util/precompile_makie_plots.jl | 13 +++------ util/precompile_plots_plots.jl | 49 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 util/precompile_plots_plots.jl diff --git a/util/precompile_makie_plots.jl b/util/precompile_makie_plots.jl index 6d9b48ff0..f693a5bbd 100644 --- a/util/precompile_makie_plots.jl +++ b/util/precompile_makie_plots.jl @@ -1,7 +1,4 @@ -# provide option of running from command line via 'julia moment_kinetics.jl' -using Pkg -Pkg.activate(".") - +using moment_kinetics using makie_post_processing # Create a temporary directory for test output @@ -23,7 +20,7 @@ input_dict = Dict("nstep"=>1, "z_discretization" => "chebyshev_pseudospectral", "vperp_ngrid" => 5, "vperp_nelement" => 1, - "vperp_bc" => "periodic", + #"vperp_bc" => "periodic", "vperp_L" => 4.0, "vperp_discretization" => "chebyshev_pseudospectral", "vpa_ngrid" => 7, @@ -49,8 +46,7 @@ input_dict = Dict("nstep"=>1, run_moment_kinetics(input_dict) -precompile_postproc_options = - moment_kinetics.makie_post_processing.generate_example_input_Dict() +precompile_postproc_options = makie_post_processing.generate_example_input_Dict() # Try to activate all plot types to get as much compiled as possible for (k,v) ∈ precompile_postproc_options @@ -59,5 +55,4 @@ for (k,v) ∈ precompile_postproc_options end end -makie_post_processing.makie_post_process(joinpath(test_output_directory, run_name), - precompile_postproc_options) +makie_post_process(joinpath(test_output_directory, run_name), precompile_postproc_options) diff --git a/util/precompile_plots_plots.jl b/util/precompile_plots_plots.jl new file mode 100644 index 000000000..33e3181af --- /dev/null +++ b/util/precompile_plots_plots.jl @@ -0,0 +1,49 @@ +using moment_kinetics +using plots_post_processing + +# Create a temporary directory for test output +test_output_directory = tempname() +run_name = "precompilation" +mkpath(test_output_directory) + +input_dict = Dict("nstep"=>1, + "run_name"=>run_name, + "base_directory" => test_output_directory, + "dt" => 0.0, + "r_ngrid" => 5, + "r_nelement" => 1, + "r_bc" => "periodic", + "r_discretization" => "chebyshev_pseudospectral", + "z_ngrid" => 5, + "z_nelement" => 1, + "z_bc" => "wall", + "z_discretization" => "chebyshev_pseudospectral", + "vperp_ngrid" => 5, + "vperp_nelement" => 1, + #"vperp_bc" => "periodic", + "vperp_L" => 4.0, + "vperp_discretization" => "chebyshev_pseudospectral", + "vpa_ngrid" => 7, + "vpa_nelement" => 1, + "vpa_bc" => "periodic", + "vpa_L" => 4.0, + "vpa_discretization" => "chebyshev_pseudospectral", + "vzeta_ngrid" => 7, + "vzeta_nelement" => 1, + "vzeta_bc" => "periodic", + "vzeta_L" => 4.0, + "vzeta_discretization" => "chebyshev_pseudospectral", + "vr_ngrid" => 7, + "vr_nelement" => 1, + "vr_bc" => "periodic", + "vr_L" => 4.0, + "vr_discretization" => "chebyshev_pseudospectral", + "vz_ngrid" => 7, + "vz_nelement" => 1, + "vz_bc" => "periodic", + "vz_L" => 4.0, + "vz_discretization" => "chebyshev_pseudospectral") + +run_moment_kinetics(input_dict) + +analyze_and_plot_data(joinpath(test_output_directory, run_name)) From 219c1159773cd2828e82f5164b66543a97487acc Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 31 Dec 2023 19:42:16 +0000 Subject: [PATCH 51/80] Use more processes for precompilation jobs PackageCompiler can now use multiple threads, so it is useful to give the precompilation jobs multiple cores. --- .../archer/jobscript-precompile-makie-post-processing.template | 2 +- machines/archer/jobscript-precompile-no-run.template | 2 +- .../archer/jobscript-precompile-plots-post-processing.template | 2 +- machines/archer/jobscript-precompile.template | 2 +- .../jobscript-precompile-makie-post-processing.template | 2 +- .../generic-batch-template/jobscript-precompile-no-run.template | 2 +- .../jobscript-precompile-plots-post-processing.template | 2 +- machines/generic-batch-template/jobscript-precompile.template | 2 +- .../marconi/jobscript-precompile-makie-post-processing.template | 2 +- machines/marconi/jobscript-precompile-no-run.template | 2 +- .../marconi/jobscript-precompile-plots-post-processing.template | 2 +- machines/marconi/jobscript-precompile.template | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/machines/archer/jobscript-precompile-makie-post-processing.template b/machines/archer/jobscript-precompile-makie-post-processing.template index 480e138f2..8e82da0c1 100644 --- a/machines/archer/jobscript-precompile-makie-post-processing.template +++ b/machines/archer/jobscript-precompile-makie-post-processing.template @@ -1,7 +1,7 @@ #!/usr/bin/env bash #SBATCH --mem=64G -#SBATCH --ntasks=1 +#SBATCH --ntasks=16 #SBATCH --cpus-per-task=1 #SBATCH --time=1:00:00 #SBATCH --account=ACCOUNT diff --git a/machines/archer/jobscript-precompile-no-run.template b/machines/archer/jobscript-precompile-no-run.template index bb444987b..3fc0465c5 100644 --- a/machines/archer/jobscript-precompile-no-run.template +++ b/machines/archer/jobscript-precompile-no-run.template @@ -1,7 +1,7 @@ #!/usr/bin/env bash #SBATCH --mem=64G -#SBATCH --ntasks=1 +#SBATCH --ntasks=16 #SBATCH --cpus-per-task=1 #SBATCH --time=1:00:00 #SBATCH --account=ACCOUNT diff --git a/machines/archer/jobscript-precompile-plots-post-processing.template b/machines/archer/jobscript-precompile-plots-post-processing.template index fdd54c4fc..543455a15 100644 --- a/machines/archer/jobscript-precompile-plots-post-processing.template +++ b/machines/archer/jobscript-precompile-plots-post-processing.template @@ -1,7 +1,7 @@ #!/usr/bin/env bash #SBATCH --mem=64G -#SBATCH --ntasks=1 +#SBATCH --ntasks=16 #SBATCH --cpus-per-task=1 #SBATCH --time=1:00:00 #SBATCH --account=ACCOUNT diff --git a/machines/archer/jobscript-precompile.template b/machines/archer/jobscript-precompile.template index e4d681ed6..f405df982 100644 --- a/machines/archer/jobscript-precompile.template +++ b/machines/archer/jobscript-precompile.template @@ -1,7 +1,7 @@ #!/usr/bin/env bash #SBATCH --mem=64G -#SBATCH --ntasks=1 +#SBATCH --ntasks=16 #SBATCH --cpus-per-task=1 #SBATCH --time=1:00:00 #SBATCH --account=ACCOUNT diff --git a/machines/generic-batch-template/jobscript-precompile-makie-post-processing.template b/machines/generic-batch-template/jobscript-precompile-makie-post-processing.template index 8a0aefa5c..c3db0bcfb 100644 --- a/machines/generic-batch-template/jobscript-precompile-makie-post-processing.template +++ b/machines/generic-batch-template/jobscript-precompile-makie-post-processing.template @@ -1,6 +1,6 @@ #!/usr/bin/env bash -#SBATCH --ntasks=1 +#SBATCH --ntasks=16 #SBATCH --time=1:00:00 #SBATCH --account=ACCOUNT #SBATCH --partition=some-serial-partition diff --git a/machines/generic-batch-template/jobscript-precompile-no-run.template b/machines/generic-batch-template/jobscript-precompile-no-run.template index 43aabcb69..55571557b 100644 --- a/machines/generic-batch-template/jobscript-precompile-no-run.template +++ b/machines/generic-batch-template/jobscript-precompile-no-run.template @@ -1,6 +1,6 @@ #!/usr/bin/env bash -#SBATCH --ntasks=1 +#SBATCH --ntasks=16 #SBATCH --time=1:00:00 #SBATCH --account=ACCOUNT #SBATCH --partition=some-serial-partition diff --git a/machines/generic-batch-template/jobscript-precompile-plots-post-processing.template b/machines/generic-batch-template/jobscript-precompile-plots-post-processing.template index 80d5736bd..b71f7e09d 100644 --- a/machines/generic-batch-template/jobscript-precompile-plots-post-processing.template +++ b/machines/generic-batch-template/jobscript-precompile-plots-post-processing.template @@ -1,6 +1,6 @@ #!/usr/bin/env bash -#SBATCH --ntasks=1 +#SBATCH --ntasks=16 #SBATCH --time=1:00:00 #SBATCH --account=ACCOUNT #SBATCH --partition=some-serial-partition diff --git a/machines/generic-batch-template/jobscript-precompile.template b/machines/generic-batch-template/jobscript-precompile.template index 0d289fc5a..fd26c88e4 100644 --- a/machines/generic-batch-template/jobscript-precompile.template +++ b/machines/generic-batch-template/jobscript-precompile.template @@ -1,6 +1,6 @@ #!/usr/bin/env bash -#SBATCH --ntasks=1 +#SBATCH --ntasks=16 #SBATCH --time=1:00:00 #SBATCH --account=ACCOUNT #SBATCH --partition=some-serial-partition diff --git a/machines/marconi/jobscript-precompile-makie-post-processing.template b/machines/marconi/jobscript-precompile-makie-post-processing.template index edbdb2c52..73ce99ad0 100644 --- a/machines/marconi/jobscript-precompile-makie-post-processing.template +++ b/machines/marconi/jobscript-precompile-makie-post-processing.template @@ -1,6 +1,6 @@ #!/usr/bin/env bash -#SBATCH --ntasks=1 +#SBATCH --ntasks=48 #SBATCH --time=1:00:00 #SBATCH --account=ACCOUNT #SBATCH --partition=skl_fua_dbg diff --git a/machines/marconi/jobscript-precompile-no-run.template b/machines/marconi/jobscript-precompile-no-run.template index b47bb7d6a..e5bbb9cc9 100644 --- a/machines/marconi/jobscript-precompile-no-run.template +++ b/machines/marconi/jobscript-precompile-no-run.template @@ -1,6 +1,6 @@ #!/usr/bin/env bash -#SBATCH --ntasks=1 +#SBATCH --ntasks=48 #SBATCH --time=1:00:00 #SBATCH --account=ACCOUNT #SBATCH --partition=skl_fua_dbg diff --git a/machines/marconi/jobscript-precompile-plots-post-processing.template b/machines/marconi/jobscript-precompile-plots-post-processing.template index ff2ea1db0..60235ad78 100644 --- a/machines/marconi/jobscript-precompile-plots-post-processing.template +++ b/machines/marconi/jobscript-precompile-plots-post-processing.template @@ -1,6 +1,6 @@ #!/usr/bin/env bash -#SBATCH --ntasks=1 +#SBATCH --ntasks=48 #SBATCH --time=1:00:00 #SBATCH --account=ACCOUNT #SBATCH --partition=skl_fua_dbg diff --git a/machines/marconi/jobscript-precompile.template b/machines/marconi/jobscript-precompile.template index ae4cbd973..651b2fc92 100644 --- a/machines/marconi/jobscript-precompile.template +++ b/machines/marconi/jobscript-precompile.template @@ -1,6 +1,6 @@ #!/usr/bin/env bash -#SBATCH --ntasks=1 +#SBATCH --ntasks=48 #SBATCH --time=1:00:00 #SBATCH --account=ACCOUNT #SBATCH --partition=skl_fua_dbg From 8762741008c1e2d58db670f20fabcd29530c5799 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Mon, 1 Jan 2024 11:55:50 +0000 Subject: [PATCH 52/80] Optionally install `Revise` and add to startup.jl Using `Revise` makes the interactive development workflow much better, so should be the default thing to do on interactive machines. --- .../compile_dependencies.sh | 4 +-- machines/generic-pc/compile_dependencies.sh | 4 +-- machines/machine_setup.sh | 4 +-- .../shared/add_dependencies_to_project.jl | 6 +++- machines/shared/machine_setup.jl | 36 +++++++++++++++++-- 5 files changed, 45 insertions(+), 9 deletions(-) diff --git a/machines/generic-batch-template/compile_dependencies.sh b/machines/generic-batch-template/compile_dependencies.sh index f4da5795a..ba52a4d75 100755 --- a/machines/generic-batch-template/compile_dependencies.sh +++ b/machines/generic-batch-template/compile_dependencies.sh @@ -19,7 +19,7 @@ ARTIFACT_DIR=$PWD ###### # Get default response for whether to download/build HDF5 -DEFAULT_BUILDHDF5=$(../../bin/julia ../shared/get_mk_preference.jl build_hdf5 "y") +DEFAULT_BUILDHDF5=$(../../bin/julia --project ../shared/get_mk_preference.jl build_hdf5 "y") if [[ $DEFAULT_BUILDHDF5 == "y" ]]; then echo "Do you want to download, and compile a local version of HDF5 (if you do" @@ -54,7 +54,7 @@ else fi # Save current response for whether to download/build HDF5 as default -../../bin/julia ../shared/set_mk_preference.jl build_hdf5 $BUILDHDF5 +../../bin/julia --project ../shared/set_mk_preference.jl build_hdf5 $BUILDHDF5 if [[ $BUILDHDF5 == "y" && -d hdf5-build ]]; then echo "HDF5 appears to have been downloaded, compiled and installed already." diff --git a/machines/generic-pc/compile_dependencies.sh b/machines/generic-pc/compile_dependencies.sh index 5a919271e..3add3b031 100755 --- a/machines/generic-pc/compile_dependencies.sh +++ b/machines/generic-pc/compile_dependencies.sh @@ -13,7 +13,7 @@ ARTIFACT_DIR=$PWD ###### # Get default response for whether to download/build HDF5 -DEFAULT_BUILDHDF5=$(../../bin/julia ../shared/get_mk_preference.jl build_hdf5 "y") +DEFAULT_BUILDHDF5=$(../../bin/julia --project ../shared/get_mk_preference.jl build_hdf5 "y") if [[ $DEFAULT_BUILDHDF5 == "y" ]]; then echo "Do you want to download, and compile a local version of HDF5 (if you do" @@ -48,7 +48,7 @@ else fi # Save current response for whether to download/build HDF5 as default -../../bin/julia ../shared/set_mk_preference.jl build_hdf5 $BUILDHDF5 +../../bin/julia --project ../shared/set_mk_preference.jl build_hdf5 $BUILDHDF5 if [[ $BUILDHDF5 == "y" && -d hdf5-build ]]; then echo "HDF5 appears to have been downloaded, compiled and installed already." diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 7695f155b..14efed581 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -246,14 +246,14 @@ echo # command, because passing as a prefix does not work (sometimes??) within a # bash script (even though as far as JTO knows it should work). export JULIA_DEPOT_PATH=$JULIA_DIRECTORY -$JULIA machines/shared/machine_setup.jl "$MACHINE" +$JULIA --project machines/shared/machine_setup.jl "$MACHINE" if [ -f julia.env ]; then # Set up modules, JULIA_DEPOT_PATH, etc. to use for the rest of this script source julia.env fi -SEPARATE_POSTPROC_PROJECTS=$(bin/julia machines/shared/get_mk_preference.jl separate_postproc_projects) +SEPARATE_POSTPROC_PROJECTS=$(bin/julia --project machines/shared/get_mk_preference.jl separate_postproc_projects) if [[ $BATCH_SYSTEM -eq 0 || $SEPARATE_POSTPROC_PROJECTS == "y" ]]; then # Batch systems can (conveniently) use different optimization flags for # running simulations and for post-processing. diff --git a/machines/shared/add_dependencies_to_project.jl b/machines/shared/add_dependencies_to_project.jl index e386fa70d..20cfcc1b7 100644 --- a/machines/shared/add_dependencies_to_project.jl +++ b/machines/shared/add_dependencies_to_project.jl @@ -66,7 +66,11 @@ for p ∈ to_rm catch end end -Pkg.add(["HDF5", "MPI", "MPIPreferences", "PackageCompiler", "SpecialFunctions"]) +to_add = ["HDF5", "MPI", "MPIPreferences", "PackageCompiler", "SpecialFunctions"] +if !mk_preferences["batch_system"] && mk_preferences["use_revise"] == "y" + push!(to_add, "Revise") +end +Pkg.add(to_add) # Instantiate packages so we can use MPIPreferences below diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index 5dc696722..7d9f19d18 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -5,6 +5,7 @@ module machine_setup export machine_setup_moment_kinetics +using Pkg using TOML # Default settings for the arguments to machine_setup_moment_kinetics(), set like this @@ -22,14 +23,16 @@ default_settings["base"] = Dict("account"=>"", "use_plots"=>"n", "separate_postproc_projects"=>"n", "use_netcdf"=>"n", - "enable_mms"=>"n") + "enable_mms"=>"n", + "use_revise"=>"n") # No batch system steup for "generic-pc" default_settings["generic-pc"] = merge(default_settings["base"], Dict("default_run_time"=>"0:00:00", "default_nodes"=>"0", "default_postproc_time"=>"0:00:00", "default_postproc_memory"=>"0", - "use_makie"=>"y")) + "use_makie"=>"y", + "use_revise"=>"y")) default_settings["generic-batch"] = deepcopy(default_settings["base"]) default_settings["archer"] = merge(default_settings["base"], Dict("default_partition"=>"standard", @@ -220,6 +223,35 @@ function machine_setup_moment_kinetics(machine::String; no_force_exit::Bool=fals get_setting("enable_mms", "Would you like to enable MMS testing?", machine, mk_preferences, ["y", "n"]) + if !batch_system + get_setting("use_revise", + "Would you like to automatically use Revise.jl (so that you do not " + * "need to restart julia after editing code)?", + machine, mk_preferences, ["y", "n"]) + if mk_preferences["use_revise"] == "y" + Pkg.add("Revise") + + # Check that `using Revise` is in the startup.jl + if julia_directory == "" + depot_directory = DEPOT_PATH[1] + else + depot_directory = julia_directory + end + # Ensure that the config/ subdirectory exists in depot_directory + config_path = joinpath(depot_directory, "config") + mkpath(config_path) + # Ensure startup.jl is a file + startup_path = joinpath(config_path, "startup.jl") + touch(startup_path) + result = run(`grep "using Revise" $startup_path`, wait=false) + if !success(result) + println("Adding `using Revise` to $startup_path") + open(startup_path, "a") do io + println(io, "\nusing Revise") + end + end + end + end # Write these preferences into a [moment_kinetics] section in LocalPreferences.toml # From 60c4bf7a243c5bd4d51b1ec57d96cccc895f1046 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Mon, 1 Jan 2024 17:12:00 +0000 Subject: [PATCH 53/80] Handle optional post-processing packages in job submission scripts If makie_post_processing is not available, use plots_post_processing; if neither is available, print a warning and do not submit a post-processing job. --- machines/shared/submit-restart.sh | 13 +++++++++++-- machines/shared/submit-run.sh | 13 +++++++++++-- util/get-job-info.jl | 2 ++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/machines/shared/submit-restart.sh b/machines/shared/submit-restart.sh index a1cf53afc..78a0a2538 100755 --- a/machines/shared/submit-restart.sh +++ b/machines/shared/submit-restart.sh @@ -17,6 +17,8 @@ POSTPROCTIME=${JOBINFO[4]} POSTPROCMEMORY=${JOBINFO[5]} PARTITION=${JOBINFO[6]} QOS=${JOBINFO[7]} +MAKIE_AVAILABLE=${JOBINFO[8]} +PLOTS_AVAILABLE=${JOBINFO[9]} # Parse command line options # [See e.g. https://www.stackchief.com/tutorials/Bash%20Tutorial%3A%20getopts @@ -36,7 +38,7 @@ Usage: submit-run.sh [option] INPUT_FILE -f JOBID Make this job start after JOBID finishes successfully -m MEM The requested memory for post-processing -n NODES The number of nodes to use for the simulation --o Use original post_processing, instead of makie_post_processing, for the post-processing job +-o Use original post_processing, instead of makie_post_processing, for the post-processing job when both are available -p PARTITION The 'partition' (passed to 'sbatch --partition') -q QOS The 'quality of service' (passed to 'sbatch --qos') -r FILE The output file to restart from (defaults to latest output in the run directory) @@ -81,6 +83,11 @@ Usage: submit-run.sh [option] INPUT_FILE esac done +if [[ "$MAKIE_AVAILABLE" == "n" && "$PLOTS_AVAILABLE" == "y" ]]; then + # No Makie post-processing available, so always use Plots post-processing + MAKIEPOSTPROCESS=0 +fi + # Get the positional argument as INPUTFILE # [See https://stackoverflow.com/a/13400237] INPUTFILE=${@:$OPTIND:1} @@ -110,7 +117,9 @@ if [[ $SUBMIT -eq 0 ]]; then echo "In the queue" > ${RUNDIR}slurm-$JOBID.out fi -if [[ $POSTPROC -eq 0 ]]; then +if [[ $POSTPROC -eq 0 && "$MAKIE_AVAILABLE" == "n" && "$PLOTS_AVAILABLE" == "n" ]]; then + echo "No post-processing packages available, so no post-processing job submitted" +elif [[ $POSTPROC -eq 0 ]]; then # Create a submission script for post-processing POSTPROCJOBSCRIPT=${RUNDIR}$RUNNAME-post.job if [[ MAKIEPOSTPROCESS -eq 1 ]]; then diff --git a/machines/shared/submit-run.sh b/machines/shared/submit-run.sh index f6fa2a6a6..dd5478955 100755 --- a/machines/shared/submit-run.sh +++ b/machines/shared/submit-run.sh @@ -17,6 +17,8 @@ POSTPROCTIME=${JOBINFO[4]} POSTPROCMEMORY=${JOBINFO[5]} PARTITION=${JOBINFO[6]} QOS=${JOBINFO[7]} +MAKIE_AVAILABLE=${JOBINFO[8]} +PLOTS_AVAILABLE=${JOBINFO[9]} # Parse command line options # [See e.g. https://www.stackchief.com/tutorials/Bash%20Tutorial%3A%20getopts @@ -35,7 +37,7 @@ Usage: submit-run.sh [option] INPUT_FILE -f JOBID Make this job start after JOBID finishes successfully -m MEM The requested memory for post-processing -n NODES The number of nodes to use for the simulation --o Use original post_processing, instead of makie_post_processing, for the post-processing job +-o Use original post_processing, instead of makie_post_processing, for the post-processing job when both are available -p PARTITION The 'partition' (passed to 'sbatch --partition') -q QOS The 'quality of service' (passed to 'sbatch --qos') -s Only create submission scripts, do not actually submit jobs @@ -76,6 +78,11 @@ Usage: submit-run.sh [option] INPUT_FILE esac done +if [[ "$MAKIE_AVAILABLE" == "n" && "$PLOTS_AVAILABLE" == "y" ]]; then + # No Makie post-processing available, so always use Plots post-processing + MAKIEPOSTPROCESS=0 +fi + # Get the positional argument as INPUTFILE # [See https://stackoverflow.com/a/13400237] INPUTFILE=${@:$OPTIND:1} @@ -105,7 +112,9 @@ if [[ $SUBMIT -eq 0 ]]; then echo "In the queue" > ${RUNDIR}slurm-$JOBID.out fi -if [[ $POSTPROC -eq 0 ]]; then +if [[ $POSTPROC -eq 0 && "$MAKIE_AVAILABLE" == "n" && "$PLOTS_AVAILABLE" == "n" ]]; then + echo "No post-processing packages available, so no post-processing job submitted" +elif [[ $POSTPROC -eq 0 ]]; then # Create a submission script for post-processing POSTPROCJOBSCRIPT=${RUNDIR}$RUNNAME-post.job if [[ MAKIEPOSTPROCESS -eq 1 ]]; then diff --git a/util/get-job-info.jl b/util/get-job-info.jl index 12fa1a128..9cf09920d 100755 --- a/util/get-job-info.jl +++ b/util/get-job-info.jl @@ -43,6 +43,8 @@ settings_string *= check_and_get_pref("default_postproc_time") settings_string *= check_and_get_pref("default_postproc_memory") settings_string *= check_and_get_pref("default_partition") settings_string *= check_and_get_pref("default_qos") +settings_string *= check_and_get_pref("use_makie") +settings_string *= check_and_get_pref("use_plots") println(settings_string) exit(0) From 89dc5dfab56193fa623967ac4bb8da068303fdca Mon Sep 17 00:00:00 2001 From: John Omotani Date: Mon, 1 Jan 2024 17:21:16 +0000 Subject: [PATCH 54/80] Add post-processing system image files to .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 5c2c8fdf2..a99a993c3 100644 --- a/.gitignore +++ b/.gitignore @@ -20,5 +20,7 @@ LocalPreferences.toml /.julia_directory_default.txt /.this_machine_name.txt moment_kinetics.so +makie_postproc.so +plots_postproc.so precompile-temp post_processing_input.toml From 86a09278515b53eef3569bfece459127142730f1 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Mon, 1 Jan 2024 17:33:35 +0000 Subject: [PATCH 55/80] Suppress error messages from Qt in plots_post_processing batch jobs Also significantly speeds up the post-processing job. --- machines/archer/jobscript-postprocess-plotsjl.template | 4 ++++ .../jobscript-postprocess-plotsjl.template | 4 ++++ machines/marconi/jobscript-postprocess-plotsjl.template | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/machines/archer/jobscript-postprocess-plotsjl.template b/machines/archer/jobscript-postprocess-plotsjl.template index 14da865f2..dba0409ff 100644 --- a/machines/archer/jobscript-postprocess-plotsjl.template +++ b/machines/archer/jobscript-postprocess-plotsjl.template @@ -19,6 +19,10 @@ source julia.env export SRUN_CPUS_PER_TASK=$SLURM_CPUS_PER_TASK echo "post-processing (with original post_processing) RUNDIR $(date)" + +# Set this environment variable to avoid warning messages from Qt when running without a display +export QT_QPA_PLATFORM=offscreen + bin/julia -Jplots_postproc.so --project=plots_post_processing/ run_post_processing.jl RUNDIR echo "finished post-processing RUNDIR $(date)" diff --git a/machines/generic-batch-template/jobscript-postprocess-plotsjl.template b/machines/generic-batch-template/jobscript-postprocess-plotsjl.template index 1bc71d243..2a89c009f 100644 --- a/machines/generic-batch-template/jobscript-postprocess-plotsjl.template +++ b/machines/generic-batch-template/jobscript-postprocess-plotsjl.template @@ -14,6 +14,10 @@ cd $SLURM_SUBMIT_DIR source julia.env echo "post-processing (with original post_processing) RUNDIR $(date)" + +# Set this environment variable to avoid warning messages from Qt when running without a display +export QT_QPA_PLATFORM=offscreen + bin/julia -Jplots_postproc.so --project=plots_post_processing/ run_post_processing.jl RUNDIR echo "finished post-processing RUNDIR $(date)" diff --git a/machines/marconi/jobscript-postprocess-plotsjl.template b/machines/marconi/jobscript-postprocess-plotsjl.template index 148c7bd3e..4655b2ea4 100644 --- a/machines/marconi/jobscript-postprocess-plotsjl.template +++ b/machines/marconi/jobscript-postprocess-plotsjl.template @@ -14,6 +14,10 @@ cd $SLURM_SUBMIT_DIR source julia.env echo "post-processing (with original post_processing) RUNDIR $(date)" + +# Set this environment variable to avoid warning messages from Qt when running without a display +export QT_QPA_PLATFORM=offscreen + bin/julia -Jplots_postproc.so --project=plots_post_processing/ run_post_processing.jl RUNDIR echo "finished post-processing RUNDIR $(date)" From c1a06d34355de0a81352ea2a7f9c3ddc7b40ab74 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 2 Jan 2024 13:49:36 +0000 Subject: [PATCH 56/80] Get MACHINE first, rather than in the middle of finding `julia` We do not need `julia` to get MACHINE, so do it first just to make the steps separate for clarity. --- machines/machine_setup.sh | 84 +++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 14efed581..3fb6c1d54 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -50,6 +50,48 @@ fi # Create directory to set up Python venv and download/compile dependencies in mkdir -p machines/artifacts +# Get name of 'machine' +while [[ -z $MACHINE || !( $MACHINE == "generic-pc" || $MACHINE == "generic-batch" || $MACHINE == "archer" || $MACHINE == "marconi" ) ]]; do + echo "Enter name of the machine to set up (must be one of 'generic-pc'," + echo "'generic-batch', 'archer', or 'marconi') [$DEFAULT_MACHINE]:" + read -p "> " MACHINE + echo + if [ -z $MACHINE ]; then + MACHINE=$DEFAULT_MACHINE + fi +done + +if [[ $MACHINE == "generic-batch" && ! -d machines/generic-batch ]]; then + echo "To use 'generic-batch' you must copy 'machines/generic-batch-template' to 'machines/generic-batch' and:" + echo "* Edit the modules in 'machines/generic-batch/julia.env' (see comments in that file)" + echo "* Edit the 'jobscript-*.template' files for precompilation or post-processing jobswith the correct serial or" + echo " debug queue for your machine." + echo "* If you want to use a system-provided HDF5 you can delete 'machines/generic-batch/compile_dependencies.sh'," + echo " and uncomment the 'hdf5_library_setting = \"system\"' option in 'machines/generic-batch/machine_settings.toml'" + echo "* If 'MPIPreferences.use_system_binary()' cannot auto-detect your MPI library and/or if 'mpirun' is not the " + echo " right command to launch MPI processes, then you need to set the 'mpi_library_names' and 'mpiexec' settings in" + echo " 'machines/generic-batch/machine_settings.toml' (note if either of these settings is set, then both must be)" + echo "* If 'mpirun' is not the right command to launch MPI processes, you may need to edit the 'jobscript-run.template'" + echo " and 'jobscript-restart.template' files in 'machines/generic-batch/' and set the setting in" + echo " 'machines/generic-batch/machine_settings.toml'" + echo "Note that 'generic-batch' is set up assuming a Linux, x86_64 based machine that uses the 'module' system and a" + echo "SLURM job queue." + exit 1 +fi + +echo "Setting up for '$MACHINE'" +echo + +# Save the machine name so we can use it as the default if we re-run +# machine_setup.sh. +echo $MACHINE > .this_machine_name.txt + +if [[ $MACHINE == "generic-pc" ]]; then + BATCH_SYSTEM=1 +else + BATCH_SYSTEM=0 +fi + # Make sure $JULIA is set # Note [ -z "$VAR" ] tests if $VAR is empty. Need the quotes to ensure that the # contents of $VAR are not evaluated if it is not empty. @@ -97,38 +139,6 @@ else fi fi -# Get name of 'machine' -while [[ -z $MACHINE || !( $MACHINE == "generic-pc" || $MACHINE == "generic-batch" || $MACHINE == "archer" || $MACHINE == "marconi" ) ]]; do - echo "Enter name of the machine to set up (must be one of 'generic-pc'," - echo "'generic-batch', 'archer', or 'marconi') [$DEFAULT_MACHINE]:" - read -p "> " MACHINE - echo - if [ -z $MACHINE ]; then - MACHINE=$DEFAULT_MACHINE - fi -done - -if [[ $MACHINE == "generic-batch" && ! -d machines/generic-batch ]]; then - echo "To use 'generic-batch' you must copy 'machines/generic-batch-template' to 'machines/generic-batch' and:" - echo "* Edit the modules in 'machines/generic-batch/julia.env' (see comments in that file)" - echo "* Edit the 'jobscript-*.template' files for precompilation or post-processing jobswith the correct serial or" - echo " debug queue for your machine." - echo "* If you want to use a system-provided HDF5 you can delete 'machines/generic-batch/compile_dependencies.sh'," - echo " and uncomment the 'hdf5_library_setting = \"system\"' option in 'machines/generic-batch/machine_settings.toml'" - echo "* If 'MPIPreferences.use_system_binary()' cannot auto-detect your MPI library and/or if 'mpirun' is not the " - echo " right command to launch MPI processes, then you need to set the 'mpi_library_names' and 'mpiexec' settings in" - echo " 'machines/generic-batch/machine_settings.toml' (note if either of these settings is set, then both must be)" - echo "* If 'mpirun' is not the right command to launch MPI processes, you may need to edit the 'jobscript-run.template'" - echo " and 'jobscript-restart.template' files in 'machines/generic-batch/' and set the setting in" - echo " 'machines/generic-batch/machine_settings.toml'" - echo "Note that 'generic-batch' is set up assuming a Linux, x86_64 based machine that uses the 'module' system and a" - echo "SLURM job queue." - exit 1 -fi - -echo "Setting up for '$MACHINE'" -echo - if [[ $DOWNLOAD_JULIA -eq 0 ]]; then # Download a version of Julia that is correct for this machine. # Here the user can specify which version of Julia they want to use in @@ -189,16 +199,6 @@ echo "Using Julia at $JULIA" echo echo "$JULIA" > .julia_default.txt -# Save the machine name so we can use it as the default if we re-run -# machine_setup.sh. -echo $MACHINE > .this_machine_name.txt - -if [[ $MACHINE == "generic-pc" ]]; then - BATCH_SYSTEM=1 -else - BATCH_SYSTEM=0 -fi - # Get the location for the .julia directory, in case this has to have a # non-default value, e.g. because the user's home directory is not accessible # from compute nodes. From 09998deed646960062ba6fa3c9bed05fa6a5c75e Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 2 Jan 2024 11:06:58 +0000 Subject: [PATCH 57/80] When submitting batch jobs, check system image is newer than source code If the system image is not newer than the source code, probably the source code has been modified since the system image was compiled. The changes to the code will not take effect until the system image is re-compiled, so print a warning advising the user to re-compile the system image. We do not error (so leave it up to the user to cancel any submitted jobs) because sometimes it might be OK to use the 'older' system image (e.g. if changes were made and then undone, or if the user really wants to run with the older code). --- machines/shared/submit-restart.sh | 9 ++++++ machines/shared/submit-run.sh | 9 ++++++ util/check_so_newer_than_code.jl | 54 +++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 util/check_so_newer_than_code.jl diff --git a/machines/shared/submit-restart.sh b/machines/shared/submit-restart.sh index 78a0a2538..b4bedec51 100755 --- a/machines/shared/submit-restart.sh +++ b/machines/shared/submit-restart.sh @@ -111,6 +111,9 @@ fi RESTARTJOBSCRIPT=${RUNDIR}$RUNNAME-restart.job sed -e "s|NODES|$NODES|" -e "s|RUNTIME|$RUNTIME|" -e "s|ACCOUNT|$ACCOUNT|" -e "s|PARTITION|$PARTITION|" -e "s|QOS|$QOS|" -e "s|RUNDIR|$RUNDIR|" -e "s|INPUTFILE|$INPUTFILE|" -e "s|RESTARTFROM|$RESTARTFROM|" machines/$MACHINE/jobscript-restart.template > $RESTARTJOBSCRIPT +# Check that source code has not been changed since moment_kinetics.so was created +bin/julia --project -O3 --check-bounds=no util/check_so_newer_than_code.jl moment_kinetics.so + if [[ $SUBMIT -eq 0 ]]; then JOBID=$(sbatch $FOLLOWFROM --parsable $RESTARTJOBSCRIPT) echo "Restart: $JOBID" @@ -124,8 +127,14 @@ elif [[ $POSTPROC -eq 0 ]]; then POSTPROCJOBSCRIPT=${RUNDIR}$RUNNAME-post.job if [[ MAKIEPOSTPROCESS -eq 1 ]]; then POSTPROCESSTEMPLATE=jobscript-postprocess.template + + # Check that source code has not been changed since makie_postproc.so was created + bin/julia --project=makie_post_processing -O3 util/check_so_newer_than_code.jl makie_postproc.so else POSTPROCESSTEMPLATE=jobscript-postprocess-plotsjl.template + + # Check that source code has not been changed since plots_postproc.so was created + bin/julia --project=plots_post_processing -O3 util/check_so_newer_than_code.jl plots_postproc.so fi sed -e "s|POSTPROCMEMORY|$POSTPROCMEMORY|" -e "s|POSTPROCTIME|$POSTPROCTIME|" -e "s|ACCOUNT|$ACCOUNT|" -e "s|RUNDIR|$RUNDIR|" machines/$MACHINE/$POSTPROCESSTEMPLATE > $POSTPROCJOBSCRIPT diff --git a/machines/shared/submit-run.sh b/machines/shared/submit-run.sh index dd5478955..ee363dd99 100755 --- a/machines/shared/submit-run.sh +++ b/machines/shared/submit-run.sh @@ -106,6 +106,9 @@ mkdir -p $RUNDIR RUNJOBSCRIPT=${RUNDIR}$RUNNAME.job sed -e "s|NODES|$NODES|" -e "s|RUNTIME|$RUNTIME|" -e "s|ACCOUNT|$ACCOUNT|" -e "s|PARTITION|$PARTITION|" -e "s|QOS|$QOS|" -e "s|RUNDIR|$RUNDIR|" -e "s|INPUTFILE|$INPUTFILE|" machines/$MACHINE/jobscript-run.template > $RUNJOBSCRIPT +# Check that source code has not been changed since moment_kinetics.so was created +bin/julia --project -O3 --check-bounds=no util/check_so_newer_than_code.jl moment_kinetics.so + if [[ $SUBMIT -eq 0 ]]; then JOBID=$(sbatch $FOLLOWFROM --parsable $RUNJOBSCRIPT) echo "Run: $JOBID" @@ -119,8 +122,14 @@ elif [[ $POSTPROC -eq 0 ]]; then POSTPROCJOBSCRIPT=${RUNDIR}$RUNNAME-post.job if [[ MAKIEPOSTPROCESS -eq 1 ]]; then POSTPROCESSTEMPLATE=jobscript-postprocess.template + + # Check that source code has not been changed since makie_postproc.so was created + bin/julia --project=makie_post_processing -O3 util/check_so_newer_than_code.jl makie_postproc.so else POSTPROCESSTEMPLATE=jobscript-postprocess-plotsjl.template + + # Check that source code has not been changed since plots_postproc.so was created + bin/julia --project=plots_post_processing -O3 util/check_so_newer_than_code.jl plots_postproc.so fi sed -e "s|POSTPROCMEMORY|$POSTPROCMEMORY|" -e "s|POSTPROCTIME|$POSTPROCTIME|" -e "s|ACCOUNT|$ACCOUNT|" -e "s|RUNDIR|$RUNDIR|" machines/$MACHINE/$POSTPROCESSTEMPLATE > $POSTPROCJOBSCRIPT diff --git a/util/check_so_newer_than_code.jl b/util/check_so_newer_than_code.jl new file mode 100644 index 000000000..e50ee19ad --- /dev/null +++ b/util/check_so_newer_than_code.jl @@ -0,0 +1,54 @@ +function check_so_newer_than_code(so_filename = "moment_kinetics.so") + is_makie = occursin("makie", so_filename) + is_plots = occursin("plots", so_filename) + + if !isfile(so_filename) + error("Trying to check age of $so_filename, but $(realpath(so_filename)) does " + * "not exist") + end + + # Get modification time of *.so system image + so_mtime = mtime(so_filename) + + # Get newest modification time of julia source file + newest_jl_mtime = 0.0 + + function get_newest_jl_mtime(directory) + for (root, dirs, files) ∈ walkdir(directory; follow_symlinks=true) + for f ∈ files + mt = mtime(joinpath(root, f)) + newest_jl_mtime = max(mt, newest_jl_mtime) + end + end + end + get_newest_jl_mtime("moment_kinetics/src/") + if is_makie + get_newest_jl_mtime("makie_post_processing/makie_post_processing/src/") + end + if is_plots + get_newest_jl_mtime("plots_post_processing/plots_post_processing/src/") + end + + so_is_newer = so_mtime > newest_jl_mtime + + if !so_is_newer + error_message = + "WARNING: source code files have been modified more recently than\n" * + "'$so_filename'. It is likely that you need to re-compile your system image\n" * + "(so that the code changes take effect) by re-running " + if is_makie + error_message *= "'precompile-makie-post-processing-submit.sh'." + elseif is_plots + error_message *= "'precompile-plots-post-processing-submit.sh'." + else + error_message *= "'precompile-submit.sh'." + end + println(error_message) + end + + return so_is_newer +end + +if abspath(PROGRAM_FILE) == @__FILE__ + check_so_newer_than_code(ARGS...) +end From 3aa9552781bbf07b73c784c02d8610ad72482273 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 2 Jan 2024 13:12:11 +0000 Subject: [PATCH 58/80] By default, submit system-image compilation for post-processing too When running `precompile-submit.sh`, submit system image compilation jobs for any post-processing package that has been set up as well. --- machines/shared/machine_setup_stage_two.jl | 2 +- machines/shared/precompile-submit.sh | 17 +++++++++++++++-- util/check_so_newer_than_code.jl | 6 ++++-- util/get-precompile-info.jl | 2 ++ 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/machines/shared/machine_setup_stage_two.jl b/machines/shared/machine_setup_stage_two.jl index 45b141a0e..2fba87d35 100644 --- a/machines/shared/machine_setup_stage_two.jl +++ b/machines/shared/machine_setup_stage_two.jl @@ -49,7 +49,7 @@ if batch_system end if mk_preferences["submit_precompilation"] == "y" - run(`./precompile-submit.sh`) + run(`./precompile-submit.sh -o`) end end diff --git a/machines/shared/precompile-submit.sh b/machines/shared/precompile-submit.sh index 1b3e9c592..dd09b0bfb 100755 --- a/machines/shared/precompile-submit.sh +++ b/machines/shared/precompile-submit.sh @@ -4,17 +4,21 @@ set -e # Parse command line options NO_PRECOMPILE_RUN=1 -while getopts "hn" opt; do +NO_POSTPROC=1 +while getopts "hno" opt; do case $opt in h) echo "Submit job to precompile moment kinetics -h Print help and exit --n No 'precompile run' - i.e. call precompile-no-run.jl instead of precompile.jl" +-n No 'precompile run' - i.e. call precompile-no-run.jl instead of precompile.jl +-o Only compile moment_kinetics.so, skipping any available post-processing packages" exit 1 ;; n) NO_PRECOMPILE_RUN=0 ;; + o) + NO_POSTPROC=0 esac done @@ -27,6 +31,8 @@ source julia.env JOBINFO=($(util/get-precompile-info.jl)) MACHINE=${JOBINFO[0]} ACCOUNT=${JOBINFO[1]} +MAKIE_AVAILABLE=${JOBINFO[2]} +PLOTS_AVAILABLE=${JOBINFO[3]} PRECOMPILEDIR=precompile-temp/ mkdir -p $PRECOMPILEDIR @@ -46,4 +52,11 @@ JOBID=$(sbatch --parsable $JOBSCRIPT) echo "Precompile: $JOBID" echo "In the queue" > $PRECOMPILEDIR/slurm-$JOBID.out +if [[ "$NO_POSTPROC" -eq 1 && "$MAKIE_AVAILABLE" == "y" ]]; then + ./precompile-makie-post-processing-submit.sh +fi +if [[ "$NO_POSTPROC" -eq 1 && "$PLOTS_AVAILABLE" == "y" ]]; then + ./precompile-plots-post-processing-submit.sh +fi + echo "Done" diff --git a/util/check_so_newer_than_code.jl b/util/check_so_newer_than_code.jl index e50ee19ad..714711f16 100644 --- a/util/check_so_newer_than_code.jl +++ b/util/check_so_newer_than_code.jl @@ -37,9 +37,11 @@ function check_so_newer_than_code(so_filename = "moment_kinetics.so") "'$so_filename'. It is likely that you need to re-compile your system image\n" * "(so that the code changes take effect) by re-running " if is_makie - error_message *= "'precompile-makie-post-processing-submit.sh'." + error_message *= "'precompile-makie-post-processing-submit.sh'" * + "\n(or 'precompile_submit.sh')." elseif is_plots - error_message *= "'precompile-plots-post-processing-submit.sh'." + error_message *= "'precompile-plots-post-processing-submit.sh'" * + "\n(or 'precompile_submit.sh')." else error_message *= "'precompile-submit.sh'." end diff --git a/util/get-precompile-info.jl b/util/get-precompile-info.jl index 0264a2400..e8a451a0a 100755 --- a/util/get-precompile-info.jl +++ b/util/get-precompile-info.jl @@ -36,6 +36,8 @@ function check_and_get_pref(name) end settings_string *= check_and_get_pref("machine") settings_string *= check_and_get_pref("account") +settings_string *= check_and_get_pref("use_makie") +settings_string *= check_and_get_pref("use_plots") println(settings_string) exit(0) From 45875aa2ec1462ed75960ad807673a3f1b8a41f1 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Mon, 1 Jan 2024 18:45:28 +0000 Subject: [PATCH 59/80] Update documentation for restructured packaging and setup --- README.md | 224 +++++++++----- docs/Project.toml | 5 +- docs/src/developing.md | 117 ++++++++ docs/src/index.md | 7 +- ...ent_kinetics.md => machine_setup_notes.md} | 0 docs/src/manual_setup.md | 76 +++++ docs/src/shared_memory_debugging.md | 2 +- machines/README.md | 283 ++++++++++-------- machines/shared/machine_setup.jl | 37 ++- .../src/makie_post_processing.jl | 16 +- moment_kinetics/Project.toml | 2 +- moment_kinetics/src/load_data.jl | 12 +- 12 files changed, 551 insertions(+), 230 deletions(-) rename docs/src/{setup_for_moment_kinetics.md => machine_setup_notes.md} (100%) create mode 100644 docs/src/manual_setup.md diff --git a/README.md b/README.md index 9fb004e90..65ae6a46c 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ instead ```bash $ git clone https://github.com/mabarnes/moment_kinetics ``` -When using https somethings (e.g. pushing to the remote repository) may require -you to use 2-factor authentication, see +When using https some things (e.g. pushing to the remote repository) may +require you to use 2-factor authentication, see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls. !!! warning @@ -25,40 +25,86 @@ https://docs.github.com/en/get-started/getting-started-with-git/about-remote-rep information from git when running the code, so without the git repository you will not be able to run a simulation. -If you are working on a supported machine, use the `machines/machine_setup.sh` -script, see [Setup for `moment_kinetics` on known clusters](@ref). Otherwise: - -1) Ensure that the Julia version is >= 1.7.0 by doing +1) If you have already installed Julia, ensure that the Julia version is >= + 1.9.0 by doing ``` $ julia --version ``` - at command line. + at command line. The setup script in step 2 can also download a Julia + binary if you have not already installed Julia. -2) Dependencies need to be installed to the project - environment. Start Julia with +2) If you are running on a desktop/laptop (rather than an HPC cluster) ensure + that you have an MPI implementation installed (using whatever the usual way + of installing software is on your system). It should not matter which MPI + implementation - `openmpi` is often a good choice if you have no reason to + prefer a particular one. Check that the MPI compiler wrapper `mpicc` is + available, e.g. ``` - $ julia --project + $ mpicc --version ``` - (which activates the 'project' in the current directory, or after starting with `julia`, in the REPL type `]` to enter `pkg>` mode, enter `activate .` and then backspace to leave `pkg>` mode). Once in the `moment_kinetics` project, enter `pkg>` mode by typing `]` and then run the command + should run without an error. + +3) Run the setup script ``` - (moment_kinetics) pkg> instantiate + $ machines/machine_setup.sh ``` - this should download and install all the dependencies. + This script will prompt you for various options. The default choices should + be sensible in most cases. On a laptop/desktop the 'name of machine to set + up' will be 'generic-pc' and will set up for interactive use. On supported + clusters, 'name of machine' will be the name of the cluster. On other + clusters 'generic-batch' can be used, but requires some manual setup (see + `machines/generic-batch-template/README.md`). + + For more information, see [`machine_setup` notes](@ref). + + If you want or need to set up 'by hand' without using + `machines/machine_setup.sh`, see [Manual setup](@ref). -3) For julia>=1.6, pre-compiling dependencies manually is not necessary any more due to improvements to the native pre-compilation, so this step can be skipped (although precompiling the whole `moment_kinetics` code may still be useful sometimes). To pre-compile a static image (`dependencies.so`) that includes most of the external packages required for running and post-processing, run +Some other notes that might sometimes be useful: + +* To speed up running scripts or the first call of `run_moment_kinetics` in a + REPL session, it is possible to compile a 'system image' + (`moment_kinetics.so`). By running + ``` + $ julia --project -O3 precompile.jl ``` - $ julia -O3 precompile_dependencies.jl + and then start Julia by running for example ``` - To use the precompiled code, add an option `-Jdependencies.so` when starting julia. - It is also possible to precompile the whole package into a static image (`moment_kinetics.so`) using + $ julia --project -O3 -Jmoment_kinetics.so + ``` + this significantly decreases the load time but prevents code changes from + taking effect when `moment_kinetics.so` is used until you repeat the + compilation of the system image. Note that this also prevents the `Revise` + package from updating `moment_kinetics` when you edit the code during and + interactive session. + + System images are created by default on HPC clusters, and are required to + use the provided `jobscript-*.template` submission scripts (used by + `submit-run.sh` and `submit-restart.sh`). This is to try and minimise the + compilation that has to be replicated on all the (possibly thousands of) + processes in a parallel run. After changing source code, you should run ``` - $ julia -O3 precompile.jl + $ precompile-submit.sh ``` - this significantly decreases the load time but prevents code changes from taking effect when `moment_kinetics.so` is used without repeating the precompilation (to use this option, add an option `-Jmoment_kinetics.so` when starting julia). + (to re-compile the `moment_kinetics.so` system image). -4) In the course of development, it is sometimes helpful to upgrade the Julia version. Upgrading the version of Julia or upgrading packages may require a fresh installation of `moment_kinetics`. To make a fresh install with the latest package versions it is necessary to remove (or rename) the `Manifest.jl` file in the main directory, and generate a new `Manifest.jl` with step 2) above. It can sometimes be necessary to remove or rename the `.julia/` folder in your root directory for this step to be successful. +* In the course of development, it is sometimes helpful to upgrade the Julia + version. Upgrading the version of Julia or upgrading packages may require a + fresh installation of `moment_kinetics`. To make a fresh install with the + latest package versions you should be able to just run + ```julia + pkg> update + ``` + (to enter 'Package mode' enter ']' at the `julia>` prompt). It might + sometimes necessary or helpful to instead remove (or rename) the + `Manifest.jl` file in the main directory, and re-run the setup from step 2) + above. It can sometimes be necessary to remove or rename the `.julia/` + directory (located by default in your home directory) to force all the + dependencies to be rebuilt. -5) One may have to set an environment variable to avoid error messages from the Qt library. If you execute the command +* When using the `Plots`-based post-processing library, one may have to set an + environment variable to avoid error messages from the Qt library. If you + execute the command ``` $ julia --project run_post_processing.jl runs/your_run_dir/ ``` @@ -80,7 +126,11 @@ To run julia with optimization, type ``` $ julia -O3 --project run_moment_kinetics.jl input.toml ``` -Note that the middle character in `-O3` is a capital letter 'O', not a zero. +Note that the middle character in `-O3` is a capital letter 'O', not a zero. (On +HPC clusters, or if you selected the "set up separate packages for post +processing" option from `machines/machine_setup.sh`, you should use `-O3 +--check-bounds=no` instead of just `-O3`, and the same in the +[Restarting](@ref) section.) Options are specified in a TOML file, e.g. `input.toml` here. The defaults are specified in `moment_kinetics_input.jl`. @@ -91,17 +141,31 @@ specified in `moment_kinetics_input.jl`. ``` $ julia -O3 --project julia> using moment_kinetics - julia> run_moment_kinetics(input) + julia> run_moment_kinetics("input.toml") ``` - where `input` is a `Dict()` containing any non-default options desired. - Input can also be loaded from a TOML file passing the filaname as a String - to the second argument, e.g. + where `input` is the name of a TOML file containing the desired options. It + is also possible to pass a `Dict()` containing any non-default options + desired, which might sometimes be useful in tests or scripts ``` - julia> run_moment_kinetics("input.toml") + julia> run_moment_kinetics(input) ``` Especially when developing the code, a lot of compilation time can be saved by using [Revise.jl](https://timholy.github.io/Revise.jl/stable/), and - re-running a test case in the REPL (without restarting `julia`). + re-running a test case in the REPL (without restarting `julia`) - this is + enabled by default when setting up using `machines/machine_setup.sh` for + 'generic-pc'. + +On an HPC cluster, you can submit a simulation (using the input file +`input.toml`) to the batch queue using the convenience script +``` +$ ./submit-run.sh input.toml +``` +See the help text +``` +$ ./submit-run.sh -h +``` +for various command line options to change parameters (e.g. number of nodes, +etc.). ### Stopping a run @@ -154,6 +218,22 @@ particular time index to restart from, e.g. julia> run_moment_kinetics("input.toml", restart="runs/example/example.dfns.h5", restart_time_index=42) ``` +On an HPC cluster, you can submit a restart (using the input file +`input.toml`) to the batch queue using the convenience script +``` +$ ./submit-restart.sh input.toml +``` +or to restart from a particular output file +``` +$ ./submit-restart.sh -r runs/example/example.dfns.h5 input.toml +``` +See the help text +``` +$ ./submit-restart.sh -h +``` +for various other command line options to change parameters (e.g. number of +nodes, etc.). + It is possible to restart a run from another output file with different resolution settings or different moment-kinetic options. This is done by interpolating variables from the old run onto the new grid. @@ -172,11 +252,29 @@ with the distributed-MPI parallelisation. When not using [Parallel I/O](@ref), the distributed-MPI domain decomposition must be identical in the old and new runs (as each block only reads from a single file). -## Post processing quickstart +## Post-processing with `makie_post_processing` + +The default post-processing module, written to be a bit more generic and +flexible than the original Plots-based one, and able to be used interactively, +is provided in `makie_post_processing`, see [Post processing](@ref). + +On an HPC cluster, when you call `./submit-run.sh` or `./submit-restart.sh`, a +job will (by default) be submitted to run +[`makie_post_processing.makie_post_process`](@ref) or +[`plots_post_processing.analyze_and_plot_data`](@ref) (depending on which you +have set up, or on whether you pass the `-o` argument when both are set up) on +the output after the run is finished. You can skip this by passing the `-a` +argument to `./submit-run.sh` or `./submit-restart.sh`. + +### Original, Plots-based post processing quickstart + +This post-processing functionality is now disabled by default, but you can +enable it by entering `y` at the "Would you like to set up +plots\_post\_processing?" prompt in `machines/machine_setup.sh`. To make plots and calculate frequencies/growth rates, run ``` -$ julia --project run_post_processing.jl runs/ +$ julia --project -O3 run_post_processing.jl runs/ ``` passing the directory to process as a command line argument. Input options for post-processing can be specified in `post_processing_input.jl`. Note that @@ -185,36 +283,22 @@ modifying `post_processing_input.jl`. Post processing can be done for several directories at once using ``` -$ julia --project post_processing_driver.jl runs/ runs/ ... +$ julia --project -O3 post_processing_driver.jl runs/ runs/ ... ``` passing the directories to process as command line arguments. Optionally pass a number as the first argument to parallelise post processing of different directories. -### Alternative post-processing - -An alternative post-processing module, written to be a bit more generic and -flexible, and able to be used interactively, is provided in -`makie_post_processing`, see [Post processing](@ref). - ## Parallel I/O -Note that to enable parallel I/O, you need to get HDF5.jl to use the system -HDF5 library (which must be MPI-enabled and compiled using the same MPI as you -run Julia with). To do this (see [the HDF5.jl -docs](https://juliaio.github.io/HDF5.jl/stable/#Using-custom-or-system-provided-HDF5-binaries)) -run (with the `moment_kinetics` project activated in Julia) -``` -using HDF5 - -HDF5.API.set_libraries!("/path/to/your/hdf5/directory/libhdf5.so", - "/path/to/your/hdf5/directory/libhdf5_hl.so") -``` -JTO also found that (on a Linux laptop) it was necessary to compile HDF5 from -source. The system-provided, MPI-linked libhdf5 depended on libcurl, and Julia -links to an incompatible libcurl, causing an error. When compiled from source -(enabling MPI!), HDF5 does not require libcurl (guess it is an optional -dependency), avoiding the problem. +To enable parallel I/O, HDF5.jl needs to be configured to use an HDF5 library +which has MPI enabled and is compiled using the same MPI as you run Julia with. +To ensure this happens, `machines/machine_setup.sh` will download the HDF5 +source code and compile a local copy of the library under `machines/artifacts`, +unless you enter `n` at the "Do you want to download, and compile a local +version of HDF5" prompt (except on known HPC clusters where an MPI-enabled HDF5 +is provided by a module - this is currently true on ARCHER2 - where the +module-provided HDF5 is used). ## Running parameter scans Parameter scans (see [Parameter scans](@ref)) can be performed by running @@ -228,7 +312,16 @@ $ julia -p 8 -O3 --project run_parameter_scan.jl path/to/scan/input.toml ## Tests There is a test suite in the `test/` subdirectory. It can be run in a few ways: -* Run using `Pkg`. Either using `pkg>` mode +* Execute some or all of the tests as a script. For example in the terminal run + ``` + $ julia -O3 --project test/runtests.jl + ``` + or in the REPL run + ``` + julia> include("test/runtests.jl") + ``` + Individual test files can also be used instead of `runtests.jl`, which runs all the tests. +* You can also run the tests using `Pkg`. Either using `pkg>` mode ``` $ julia -O3 --project julia> @@ -244,24 +337,14 @@ There is a test suite in the `test/` subdirectory. It can be run in a few ways: ``` julia -O3 --project -e "import Pkg; Pkg.test()` ``` -* Execute some or all of the tests as a script. For example in the terminal run - ``` - $ julia -O3 --project test/runtests.jl - ``` - or in the REPL run - ``` - julia> include("test/runtests.jl") - ``` - Individual test files can also be used instead of `runtests.jl`, which runs all the tests. + The downside of this method is that it will cause `NCDatasets` to be + installed if you did not install it already, which might sometimes cause + linking errors (related to the HDF5 library, see [Optional + dependencies](@ref)). By default the test suite should run fairly quickly (in a few minutes). To do so, it skips many cases. To run more comprehensive tests, you can activate the `--long` option: -* Using `test_args` argument - ``` - julia> Pkg.test(; test_args=["--long"]) - ``` - Note the semicolon is necessary. * In the REPL, run ``` julia> push!(ARGS, "--long") @@ -271,6 +354,11 @@ so, it skips many cases. To run more comprehensive tests, you can activate the ``` $ julia -O3 --project --long test/runtests.jl ``` +* Using `test_args` argument + ``` + julia> Pkg.test(; test_args=["--long"]) + ``` + Note the semicolon is necessary. To get more output on what tests were successful, an option `--verbose` (or `-v`) can be passed in a similar way to `--long` (if any tests fail, the output diff --git a/docs/Project.toml b/docs/Project.toml index ab9ea9a8f..80fb0c19d 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,8 +1,9 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -makie_post_processing = "14778871-8d0d-429c-ae6e-bde7d57d7f1e" +Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" +makie_post_processing = "4dd1b173-c370-4c56-9cc2-d797e41ae9f0" moment_kinetics = "b5ff72cc-06fc-4161-ad14-dba1c22ed34e" -plots_post_processing = "6f98f6d0-83ae-4c2a-9d92-db6b25a4b00d" +plots_post_processing = "c3e9b6e6-652c-46aa-9ac0-1bc582aad122" [compat] Documenter = "0.27" diff --git a/docs/src/developing.md b/docs/src/developing.md index 5f2099500..6df59a8eb 100644 --- a/docs/src/developing.md +++ b/docs/src/developing.md @@ -104,3 +104,120 @@ end ``` Internally, when the `begin_*_region()` functions need to change the region type (i.e. the requested region is not already active), they call `_block_synchronize()`, which calls `MPI.Barrier()`. They also switch over the `LoopRanges` struct contained in `looping.loop_ranges` as noted above. For optimization, the `_block_synchronize()` call can be skipped - when it is correct to do so - by passing the argument `no_synchronize=true` (or some more complicated conditional expression if synchronization is necessary when using some options but not for others). + + +## Package structure + +The structure of the packages in the `moment_kinetics` repo is set up so that +some features, which depend on 'heavy' external packages (such as `Makie`, +`Plots`, and `Symbolics`, which take a long time to precompile and load) can be +optional. + +The structure is set up by the `machines/machine_setup.sh` script, which +prompts the user for input to decide which optional components to include (as +well as some settings related to batch job submission on HPC clusters). +`machine_setup.sh` calls several other scripts to do the setup (written as far +as possible in Julia). The structure of these scripts is explained in +[`machine_setup` notes](@ref). + +The intention is that a top-level 'project' (defined by a `Project.toml` file, +which is created and populated by `machines/machine_setup.sh`) is set up in the +top-level directory of the repository. The `moment_kinetics` package itself +(which is in the `moment_kinetics/` subdirectory, defined by its own +`Project.toml` file which is tracked by git), and optionally other +post-processing packages, are added to this top-level project using +`Pkg.develop()`. + +### Optional dependencies + +Some capabilities that require optional dependencies are provided using +'package extensions' ([a new feature of Julia in +v1.9.0](https://julialang.org/blog/2023/04/julia-1.9-highlights/#package_extensions)). + +The way we use package extensions is a bit of a hack. Extensions are intended +to be activated when an optional dependency (called a 'weakdep' by Julia) is +loaded, e.g. `using moment_kinetics, NCDatasets`. This usage pattern is not the +most convenient for the way we use `moment_kinetics` where we would rather just +load `moment_kinetics` and then specify for example `binary_format = "netcdf"` +in the input TOML file. To work around this, the optional dependencies are +loaded automatically if they are installed (by calling `Base.requires()` in the +`__init__()` function of an appropriate sub-module). This is not the way +package extensions were intended to be used, and it may be a bit fragile - at +the time of writing in January 2024 there would be an error on precompilation +if the optional dependencies were added in one order, which went away when the +order was reversed. If this causes problems, we might need to consider an +alternative, for example adding the optional dependencies to the `startup.jl` +file, instead of trying to auto-load them from within the `moment_kinetics` +package. + +The optional capabilities at the moment are: +* Method of manufactured solutions (MMS) testing - this requires the + `Symbolics` package which is heavy and has a large number of dependencies. It + is convenient not to require `Symbolics` when MMS capability is not being + used. The functionality is provided by the `manufactured_solns_ext` + extension. The extension also requires the `IfElse` package, which is not + needed elsewhere in `moment_kinetics` and so is included as a 'weakdep' + although `IfElse` is not a heavy dependency. +* NetCDF output - this requires the `NCDatasets` package. Although not as heavy + as `Symbolics` or the plotting packages, NetCDF output is not required and + not used by default, so it does not hurt to make the dependency optional. As + a bonus, importing `NCDatasets` can sometimes cause linking errors when a + local or system installation of HDF5 (i.e. one not provided by the Julia + package manager) is used, as `NCDatasets` (sometimes?) seems to try to link a + different version of the library. These errors can be avoided by not enabling + NetCDF outut (when HDF5 output is preferred), or allowing Julia to use the + HDF5 library provided by its package manager (when NetCDF is preferred, + although this would mean that parallel I/O functionality is not available). + +### Post processing packages + +Post processing functionality is provided by separate packages +(`makie_post_processing` and `plots_post_processing`) rather than by +extensions. Extensions are not allowed to define new modules, functions, etc. +within the main package, they can only add new methods (i.e. new +implementations of the function for a different number of arguments, or +different types of the arguments) to functions already defined in the main +package. For post-processing, we want to add a lot of new functions, so to use +extensions instead of separate packages we would need to define all the +function names in the main package, and then separately the implementations in +the extension, which would be inconvenient and harder to maintain. + +There are two suggested ways of setting up the post-processing packages: + +1. For interactive use/development on a local machine, one or both + post-processing packages can be added to the top-level project using + `Pkg.develop()`. This is convenient as there is only one project to deal + with. Both simulations and post-processing are run using + ``` + $ bin/julia --project -O3 <...> + ``` +2. For optimized use on an HPC cluster it is better to set up a separate + project for the post-processing package(s). This allows different + optimization flags to be used for running simulations (`-O3 + --check-bounds=no`) and for post-processing (`-O3`). [Note, in particular + Makie.jl can have performance problems if run with `--check-bounds=no`, see + [here](https://github.com/MakieOrg/Makie.jl/issues/3132).] Simulations + should be run with + ``` + $ bin/julia --project -O3 --check-bounds=no <...> + ``` + and post-processing with + ``` + $ bin/julia --project=makie_post_processing -O3 <...> + ``` + or + ``` + $ bin/julia --project=plots_post_processing -O3 <...> + ``` + This option can also be used on a local machine, if you want to optimise + your simulation runs as much as possible by using the `--check-bounds=no` + flag. To do this answer `y` to the prompt "Would you like to set up separate + packages for post processing..." from `machines/machine_setup.sh`. + +To support option 2, the post-processing packages are located in +sub-sub-directories (`makie_post_processing/makie_post_processing/` and +`plots_post_processing/plots_post_processing/`), so that the separate projects +can be created in the sub-directories (`makie_post_processing/` and +`plots_post_processing`). `moment_kinetics` and the other dependencies must +also be added to the separate projects (the `machine_setup.sh` script takes +care of this). diff --git a/docs/src/index.md b/docs/src/index.md index 321d58bad..4d470146e 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,21 +1,22 @@ # Home -Documentation for moment_kinetics.jl +Documentation for moment\_kinetics.jl ## Contents ```@contents Pages = ["getting_started.md", - "machine_setup.md", "input_options.md", "post_processing_notes.md", "moment_kinetic_equations.md", "moment_constraints_notes.md", "boundary_conditions_notes.md", "external_sources_notes.md", + "debugging-hints.md", "parameter_scans.md", "developing.md", - "debugging-hints.md", + "manual_setup.md", + "machine_setup_notes.md", ] ``` diff --git a/docs/src/setup_for_moment_kinetics.md b/docs/src/machine_setup_notes.md similarity index 100% rename from docs/src/setup_for_moment_kinetics.md rename to docs/src/machine_setup_notes.md diff --git a/docs/src/manual_setup.md b/docs/src/manual_setup.md new file mode 100644 index 000000000..410a2659d --- /dev/null +++ b/docs/src/manual_setup.md @@ -0,0 +1,76 @@ +# Manual setup + +If you want or need to set up `moment_kinetics` without using +`machines/machine_setup.sh`, you will need to follow at least the steps in the +following sections. + +## Install Julia + +Download Julia from , and add it to your +`$PATH` so you can execute it from the command line. + +## Add `moment_kinetics` packages + +Create a 'project' in the top-level directory by creating an empty +`Project.toml` file, e.g. +``` +$ touch Project.toml +``` +Without this `Project.toml` file, running `julia --project` will activate a +global project, not one linked specificially to the repo you are in, which is +likely to cause confusion if you have more than one copy of the +`moment_kinetics` repo (experience suggests you are likely to end up with +multiple copies eventually!). + +To add the `moment_kinetics` package to your project, start Julia, enter +'Package mode' by pressing ']' at the prompt and use `develop` (to exit +'Package mode' and return to the usual `julia>` prompt, press backspace): +```julia +$ julia +julia> ] +pkg> develop ./moment_kinetics/ +``` +To allow post-processing add one or both of the post processing packages +```julia +pkg> develop ./makie_post_processing/makie_post_processing/ +``` +and/or +```julia +pkg> develop ./plots_post_processing/plots_post_processing/ +``` + +## Set up MPI + +You probably want to use your system's MPI rather than a Julia-provided +version. To do this add (in 'Package mode') the `MPIPreferences` package +([documentation +here](https://juliaparallel.org/MPI.jl/stable/configuration/#using_system_mpi)) +and then use its `use_system_binary()` function. +```julia +pkg> add MPIPreferences +pkg> +julia> MPIPreferences.use_system_binary() +``` +Normally this should 'just work'. Sometimes, for example if the MPI library +file is named something other than `libmpi.so`, you might have to pass some +keyword arguments to `use_system_binary()` - see +. + +## Link HDF5 + +To enable parallel I/O, you need to get HDF5.jl to use the system HDF5 library +(which must be MPI-enabled and compiled using the same MPI as you run Julia +with). To do this (see [the HDF5.jl +docs](https://juliaio.github.io/HDF5.jl/stable/#Using-custom-or-system-provided-HDF5-binaries)) +add the `HDF5` package and use its `HDF5.API.set_libraries!()` function +```julia +pkg> add HDF5 +pkg> +julia> using HDF5 +julia> HDF5.API.set_libraries!("/path/to/your/hdf5/directory/libhdf5.so", "/path/to/your/hdf5/directory/libhdf5_hl.so") +``` +JTO also found that (on a Linux laptop) it was necessary to compile HDF5 from +source. The system-provided, MPI-linked libhdf5 depended on libcurl, and Julia +links to an incompatible libcurl, causing an error. When compiled from source +(enabling MPI!), HDF5 does not require libcurl (guess it is an optional +dependency), avoiding the problem. diff --git a/docs/src/shared_memory_debugging.md b/docs/src/shared_memory_debugging.md index 51c2b2f6c..e9386ae68 120000 --- a/docs/src/shared_memory_debugging.md +++ b/docs/src/shared_memory_debugging.md @@ -1 +1 @@ -../../debug_test/README.md \ No newline at end of file +../../moment_kinetics/debug_test/README.md \ No newline at end of file diff --git a/machines/README.md b/machines/README.md index 01031eda2..0024f305d 100644 --- a/machines/README.md +++ b/machines/README.md @@ -1,143 +1,166 @@ -Setup for `moment_kinetics` on known clusters -============================================= +`machine_setup` notes +===================== -This subdirectory provides scripts to set up Julia and `moment_kinetics` to run -on some known clusters. +The `machines` subdirectory provides scripts to set up Julia and +`moment_kinetics` to run on laptops/desktops or on clusters. If the cluster is +not one of the currently supported machines, then some additional manual setup +is required. Currently supported: -* "archer" ([ARCHER2](https://www.archer2.ac.uk/)) -* "marconi" (the EUROfusion supercomputer - [Marconi](https://wiki.u-gov.it/confluence/display/SCAIUS/UG3.1%3A+MARCONI+UserGuide)) +* "generic-pc" - A generic personal computer (i.e. laptop or desktop machine). + Set up for interactive use, rather than for submitting jobs to a batch queue. +* "generic-batch" - A generic cluster using a batch queue. Requires some manual setup + first, see `machines/generic-batch-template/README.md`. +* "archer" - the UK supercomputer [ARCHER2](https://www.archer2.ac.uk/) +* "marconi" - the EUROfusion supercomputer + [Marconi](https://wiki.u-gov.it/confluence/display/SCAIUS/UG3.1%3A+MARCONI+UserGuide) -Quickstart ----------- +The usage is described in [Getting started](@ref). This page contains some +extra technical information. -From the top-level of the `moment_kinetics` repo, run -```shell -$ machines/machine_setup.sh [] -``` -If you omit the `` argument, you will be prompted to -enter it, with a default taken from your `$PATH`. If there is no `julia` in -your `$PATH` you will be asked if you want the script to download Julia for -you, or you can force the download by passing the `-d` flag (and do not pass a -``). - -The script will prompt you for several settings, with sensible defaults (where -possible). Note that some settings are needed on all machines - if a setting is -not needed it will be ignored. +## Notes on some prompts from the script You will be prompted to enter a location for your `.julia` directory. If you -are installing on a cluster which allows access to your home directory from -compute nodes, it is fine to leave this as the default. If not (e.g. on -ARCHER2), you need to set a path which is accessible from the compute nodes. -If you want to create a completely self-contained install (e.g. for -reproducibility or for debugging some dependency conflicts), you might want to -put `.julia` within the `moment_kinetics` directory (e.g. enter `.` at the -prompt). - -## After editing source code - -If you use the precompiled `moment_kinetics.so` system image, you need to -recompile it after editing source code (otherwise simulations will continue to -use the old source code). - -To do this, run -```shell -$ ./precompile-submit.sh -``` -and wait for the resulting job to complete. - -This step is required if you use the `sumbit-run.sh` or `submit-restart.sh` -scripts, as these both use the `moment_kinetics.so` system image. - -## Running and restarting simulations - -Convenience scripts are provided to submit jobs running and post-processing a -simulation run or restart. -```shell -$ ./submit-run.sh .toml -``` -will submit a job to run a simulation using that input file and (by default) a -linked job that will run the post-processing routines (in the serial queue) -when the run finishes. - -Similarly -```shell -$ ./submit-restart.sh .toml -``` -will submit a job to run and post-process a restart using input file. The -simulation will restart from the last time point of the previous run -(`run_moment_kinetics.jl` supports more flexibility, but for now you would -need to write your own submission script to pass the options needed for that). - -Default parameters for the runs (number of nodes, time limit, etc.) were set up -by `machines/machine_setup.sh` are stored in `LocalPreferences.toml` (which can -be edited to change them). The parameters can be altered for a particular job -(and you can disable the post-processing job) using command line flags -described by the help text -```shell -$ ./submit-run.sh -h -``` -or -```shell -$ ./submit-restart.sh -h -``` - -Advanced usage --------------- +are installing on a personal computer or on a cluster which allows access to +your home directory from compute nodes, it is fine to leave this as the +default. If not (e.g. on ARCHER2), you need to set a path which is accessible +from the compute nodes. If you want to create a completely self-contained +install (e.g. for reproducibility or for debugging some dependency conflicts), +you might want to put `.julia` within the `moment_kinetics` directory (i.e. +enter `.julia` at the prompt). + +## Defaults for prompts + +The default value for each of the settings that the user is prompted for +interactively are first taken from some sensible, machine-specific defaults. +When `machines/machine_setup.sh` is run, the values chosen by the user are +saved in the `[moment_kinetics]` section of `LocalPreferences.toml`, and these +values are used as the defaults next time `machines/machine_setup.sh` is run, +in order to make it easier to re-run the setup, e.g. because some dependencies +need updating, or to change just one or a few settings. + +A few settings (which are needed before Julia can be started for the first +time) are saved into hidden files (`.julia_default.txt`, +`.this_machine_name.txt`, and `.julia_directory_default.txt`) instead of into +`LocalPreferences.toml`, to avoid needing to parse a TOML file using `bash`. + +## `bin/julia` + +A symlink or script is created at `bin/julia` to call the chosen julia +executable. On HPC systems we create a file `julia.env` which must be +`source`'d (to load the right modules, etc.) before `julia` can be used - in +this case `julia.env` can be used to set up any environment variables, etc. so +a symlink is sufficient. On laptops/desktops that will be used interactively, +it is inconvenient to have to remember to `source julia.env`, especially if you +have multiple instances of `moment_kinetics`, so instead the necessary setup +(of the `JULIA_DEPOT_PATH` environment variable, if needed, and a Python venv +if the Plots-based post processing is enabled) is done by making `bin/julia` a + small bash script, which does that setup and then calls the chosen `julia` + executable, passing through to it any command line arguments. + +## `julia.env` + +A `julia.env` file is used on HPC systems to set up the environment (modules +and environment variables). On laptop/desktop systems this would be +inconvenient (especially if there are multiple instances of `moment_kinetics`) +so a `julia.env` is not used for these. + +The `julia.env` is created from a template `julia.env` which is located in the +subdirectory of `machines/` for the specific machine being set up. + +If you need to run `julia` interactively (for `moment_kinetics`) on a machine +that uses `julia.env`, either run `source julia.env` in each terminal session +where you want to use `moment_kinetics`, or add it to your `.bashrc` (if this +does not conflict with any other projects). + +!!! warning + Note that `julia.env` runs `module purge` to remove any already loaded + modules (to get a clean environment). It is therefore very likely to + interfere with other projects. + +## Setup of post processing packages + +See [Post processing packages](@ref). + +## Use of system images + +On HPC clusters, creating system images `moment_kinetics.so` and for post +processing `makie_postproc.so` and/or `plots_postproc.so` is required. This is +to avoid (as far as practical) wasting CPU hours doing identical compilation in +large parallel jobs. If you wanted to remove this requirement for some reason +(although this is not recommended), you would need to go to the subdirectory of +`machines/` for the machine you are working on, and edit the +`jobscript-run.template`, `jobscript-restart.template`, +`jobscript-postprocess.template`, and `jobscript-postprocess-plotsjl.template` +files to remove the `-J*.so` argument. If you do do this, please do not commit +those changes and merge them to the `master` branch of `moment_kinetics`. + +## Operations done by `machines/machine_setup.sh` The convenience script `machine_setup.sh` is provide because the actual setup happens in multiple stages, with Julia being restarted in between (as this is -required on some machines): -1. Start Julia (using the Julia executable you want to use in the end for - simulations) from the top level of the `moment_kinetics` repo, but not - requiring the `--project` flag; run - `include("machines/shared/machine_setup.jl")`; call the - [`moment_kinetics.machine_setup_moment_kinetics`](@ref) function with - appropriate settings as arguments (see the docstring). - * This function requires no packages outside the base Julia system image - (because on some machines it is desirable not to have to install any - packages before setting `JULIA_DEPOT_PATH`, which can be done by this - function). - * It saves some settings into `LocalPreferences.toml`; makes a symlink at - `bin/julia` to the Julia executable being used; saves environment setup - into the `julia.env` file (which can include setting the - `JULIA_DEPOT_PATH` environment variable to ensure the `.julia` - directory is accessible on compute nodes); and if necessary symlinks a - machine-specific file from - `machines/shared/machine_setup_stage_two.jl`. - * Usually Julia needs to be restarted after running this function, so it - will call `exit()` to stop Julia. This can be avoided by passing the - `no_force_exit=true` argument if that is useful. -2. Run `source julia.env` in each terminal session where you want to use - `moment_kinetics`, or add it to your `.bashrc` (if this does not conflict - with any other projects). - * Note that `julia.env` runs `module purge` to remove any already loaded - modules (to get a clean environment). It is therefore very likely to - interfere with other projects. -3. If it is necessary to run a second stage of setup, for example after Julia - is restarted with a special `JULIA_DEPOT_PATH`, start Julia again, this time - with the `--project` flag. A symlink has been created, so you can use - ```shell - $ bin/julia --project - ``` - and run `include("machines/shared/machine_setup_stage_two.jl")`. - * This script sets up MPI and HDF5. - * It exits Julia when it is finished, as Julia must be restarted when - `MPIPreferences` settings are changed. -4. If you want to build a precompiled system image (to speed up startup of - simulation runs), run - ```shell - $ precompile-submit.sh - ``` - which will submit a serial or debug job that runs `precompile.jl` to create - the `moment_kinetics.so` image. - * This is required if using the `submit-run.sh` script, unless you edit - the jobscript templates in the appropriate subdirectory - `machines/`. - -API documentation ------------------ +required on some machines), and because the script is able to download Julia if +Julia is not already installed. + +The steps done by `machines/machine_setp.sh` are: +1. Get the name of the 'machine' ('generic-pc', 'archer', etc.) so that + machine-dependent setup can be done and machine-specific defaults can be + used. () +1. Download a Julia executable, or prompt the user for the location of one + (defaulting to any `julia` found in `$PATH`). +1. Get the location of the `.julia` directory to be used by (this copy of) + `moment_kinetics`. At this point we have enough setup to start using + `julia`. Export `JULIA_DEPOT_PATH` so that this is used any time `julia` is + run in the rest of the script. +1. Run `machines/shared/machine_setup.jl`. This script (whose functions are + documented in [API documentation](@ref machine_setup_api_documentation)): + * prompts the user for most of the settings (and saves them to + `LocalPreferences.toml` from where they can be accessed by other scripts + later and used as defaults if `machines/machine_setup` is re-run) + * creates the `julia.env` file (from the template for the given machine) on + HPC systems + * creates the `bin/julia` symlink or script (see [bin/julia](@ref)) + * creates a link to the `compile_dependencies.sh` script for the machine (if + there is one). + * installs the `Revise` package and adds `using Revise` to the `startup.jl` + file (on laptop/desktop systems, and if the user did not de-select this). + + It is necessary to restart `julia` after this script has been run, so that + we can first `source julia.env` (if it exists) or use the script at + `bin/julia` in order to use the environment settings in them. +1. If `julia.env` exists, run `source julia.env` to load modules, etc. +1. Set the optimization flags that will be used for running simulations or for + running post processing. These need to be set the same as will be used + eventually so that precompilation of dependencies and packages that happens + while running `machines/machine_setup.sh` does not need to be overwritten + (due to different optimization flags) later, as this would be a waste of + time (although it should work fine). +1. Add various dependencies to the top-level project, by calling + `machines/shared/add_dependencies_to_project.jl`. This will set up MPI and + HDF5 to link to the correct libraries. `julia` needs to be restarted after + the setup of MPI and HDF5 is done, which is why this is a separate script + from the following one (this separation also allows + `add_dependencies_to_project.jl` to be re-used in + `makie_post_processing_setup.jl` and `plots_post_processing_setup.jl` if + these are to be set up as separate projects from the top-level one). +1. Complete the setup by running `machines/shared/machine_setup_stage_two.jl`, + which creates a Python venv with matplotlib installed (if + `plots_post_processing` is enabled), creates symlinks at the top level to + scripts to submit batch jobs (if setting up for an HPC cluster), and submits + a job to compile a system image for `moment_kinetics` (if setting up for an + HPC cluster, and if the user did not de-select this). +1. Set up `makie_post_processing` (if enabled) by running + `machines/shared/makie_post_processing_setup.jl` and/or + `plots_post_processing (if enabled)` by running + `machines/shared/plots_post_processing_setup.jl`. These scripts also submit + jobs to create system images for `makie_post_processing` or + `plots_post_processing` (if setting up for an HPC cluster, and if the user + did not de-select this). +1. Print a message indicating which optimization flags to use for running + simulations or for post-processing. + +[API documentation](@id machine_setup_api_documentation) +-------------------------------------------------------- ```@autodocs Modules = [moment_kinetics.machine_setup] diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index 7d9f19d18..5e9dfe075 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -41,6 +41,10 @@ default_settings["marconi"] = merge(default_settings["base"], Dict("default_partition"=>"skl_fua_prod", "default_qos"=>"normal")) """ + get_user_input(possible_values, default_value) + +Prompt for user input. If the user enters nothing, return `default_value`. Check that the +entered value is one of `possible_values`, if not prompt again. """ function get_user_input(possible_values, default_value) setting = default_value @@ -71,9 +75,13 @@ function get_user_input(possible_values, default_value) end """ + get_setting(setting_name, message, machine, local_defaults, + possible_values=nothing) -Prompt the user to set a setting. Default value is read from LocalPreferences.toml if it -has been set before, or from sensible defaults otherwise. +Prompt the user to set a setting called `setting_name` after printing `message`. Default +value is read from `local_defaults` if it exists there (which it will do if it has been +set before, as then it is stored in `LocalPreferences.toml`), or from sensible defaults in +the `machine` section of `default_settings` otherwise. """ function get_setting(setting_name, message, machine, local_defaults, possible_values=nothing) @@ -116,32 +124,35 @@ used values): get them for the current session) or in your `.bashrc` (to get them by default). Note that this calls `module purge` so will remove any currently loaded modules when it is run. -* Makes a symlink to the Julia exeutable used to run this command at `bin/julia` under - the moment_kinetics repo, so that setup and job submission scripts can use a known - relative path. +* Makes a symlink to, or a bash script that calls, the Julia executable used to run this + command at `bin/julia` under the moment_kinetics repo, so that setup and job submission + scripts can use a known relative path. !!! note If you change the Julia executable, e.g. to update to a new verison, you will need - to either replace the symlink `/bin/julia` by hand, or re-run - this function using the new executable. + to either replace the symlink `/bin/julia` or edit the bash script + at `/bin/julia` by hand, or re-run this function using the new + executable. Usually it is necessary for Julia to be restarted after running this function to run Julia -with the correct JULIA_DEPOT_PATH, etc. so the function will force Julia to exit. If for +with the correct `JULIA_DEPOT_PATH`, etc. so the function will force Julia to exit. If for some reason this is not desired (e.g. when debugging), pass `no_force_exit=true`. The `interactive` argument exists so that when this function is called from another script, terminal output with instructions for the next step can be disabled. Currently supported machines: -* `"generic-pc"` - A generic personal computer. Set up for interactive use, rather than - for submitting jobs to a batch queue. +* `"generic-pc"` - A generic personal computer (i.e. laptop or desktop machine).. Set up + for interactive use, rather than for submitting jobs to a batch queue. +* `"generic-batch"` - A generic cluster using a batch queue. Requires some manual setup + first, see `machines/generic-batch-template/README.md`. * `"archer"` - the UK supercomputer [ARCHER2](https://www.archer2.ac.uk/) * `"marconi"` - the EUROfusion supercomputer [Marconi](https://wiki.u-gov.it/confluence/display/SCAIUS/UG3.1%3A+MARCONI+UserGuide) !!! note - The settings created by this function are saved in LocalPreferences.toml (using the - `Preferences.jl` package). It might sometimes be useful to edit these by hand (e.g. - the `account` setting if this needs to be changed.): it is fine to do this. + The settings created by this function are saved in LocalPreferences.toml. It might + sometimes be useful to edit these by hand (e.g. the `account` setting if this needs + to be changed.): it is fine to do this. """ function machine_setup_moment_kinetics(machine::String; no_force_exit::Bool=false, interactive::Bool=true) diff --git a/makie_post_processing/makie_post_processing/src/makie_post_processing.jl b/makie_post_processing/makie_post_processing/src/makie_post_processing.jl index d78c73be3..54fa303fb 100644 --- a/makie_post_processing/makie_post_processing/src/makie_post_processing.jl +++ b/makie_post_processing/makie_post_processing/src/makie_post_processing.jl @@ -19,6 +19,8 @@ export animate_f_unnorm_vs_vpa, animate_f_unnorm_vs_vpa_z, get_1d_ax, get_2d_ax, include("shared_utils.jl") +# Need this import just to allow links in the docstrings to be understood by Documenter.jl +import moment_kinetics using moment_kinetics.analysis: analyze_fields_data, check_Chodura_condition, get_r_perturbation, get_Fourier_modes_2D, get_Fourier_modes_1D, steady_state_residuals, @@ -421,12 +423,12 @@ Set up input, storing in the global [`input_dict`](@ref) and [`input_dict_dfns`] be used in the various plotting and analysis functions. The `run_info` that you are using (as returned by -[`moment_kinetics.load_data.get_run_info`](@ref)) should be passed -to `run_info_moments` (if it contains only the moments), or `run_info_dfns` (if it also -contains the distributions functions), or both (if you have loaded both sets of output). -This allows default values to be set based on the grid sizes and number of time points -read from the output files. Note that `setup_makie_post_processing_input!()` is called by -default at the end of `get_run_info()`, for conveinence in interactive use. +[`get_run_info`](@ref)) should be passed to `run_info_moments` (if it contains only the +moments), or `run_info_dfns` (if it also contains the distributions functions), or both +(if you have loaded both sets of output). This allows default values to be set based on +the grid sizes and number of time points read from the output files. Note that +`setup_makie_post_processing_input!()` is called by default at the end of +`get_run_info()`, for conveinence in interactive use. By default an error is raised if `input_file` does not exist. To continue anyway, using default options, pass `allow_missing_input_file=true`. @@ -741,7 +743,7 @@ as offsets from the final time index of the run. `setup_makie_post_processing_input!()` is called at the end of `get_run_info()`, for convenience when working interactively. Use -[moment_kinetics.load_data.get_run_info_no_setup](@ref) if you do not want this. A +[`moment_kinetics.load_data.get_run_info_no_setup`](@ref) if you do not want this. A post-processing input file can be passed to `setup_input_file` that will be passed to `setup_makie_post_processing_input!()` if you do not want to use the default input file. """ diff --git a/moment_kinetics/Project.toml b/moment_kinetics/Project.toml index 4bfffe581..06812ca91 100644 --- a/moment_kinetics/Project.toml +++ b/moment_kinetics/Project.toml @@ -49,7 +49,7 @@ manufactured_solns_ext = ["Symbolics", "IfElse"] [compat] HDF5_jll = "<1.14, >=1.15" -julia = "1.7.0" +julia = "1.9.0" [extras] Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/moment_kinetics/src/load_data.jl b/moment_kinetics/src/load_data.jl index a312d2d56..64066c8bd 100644 --- a/moment_kinetics/src/load_data.jl +++ b/moment_kinetics/src/load_data.jl @@ -2307,10 +2307,11 @@ functions files. The `itime_min`, `itime_max` and `itime_skip` options can be used to select only a slice of time points when loading data. In `makie_post_process` these options are read from the -input (if they are set) before `get_run_info()` is called, so that the `run_info` returned -can be passed to [`setup_makie_post_processing_input!`](@ref), to be used for defaults for -the remaining options. If either `itime_min` or `itime_max` are ≤0, their values are used -as offsets from the final time index of the run. +input (if they are set) before `get_run_info_no_setup()` is called, so that the `run_info` +returned can be passed to +`makie_post_processing.setup_makie_post_processing_input!()`, to be used for +defaults for the remaining options. If either `itime_min` or `itime_max` are ≤0, their +values are used as offsets from the final time index of the run. """ function get_run_info_no_setup(run_dir::Union{AbstractString,Tuple{AbstractString,Union{Int,Nothing}}}...; itime_min=1, itime_max=0, itime_skip=1, dfns=false) @@ -2538,7 +2539,8 @@ end Load a variable -`run_info` is the information about a run returned by [`get_run_info`](@ref). +`run_info` is the information about a run returned by +`makie_post_processing.get_run_info()`. `variable_name` is the name of the variable to load. From f734d8d3d7db66e01a8a68001ae12b7f362ce101 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 2 Jan 2024 16:01:48 +0000 Subject: [PATCH 60/80] Print finish time in precompile jobs --- .../archer/jobscript-precompile-makie-post-processing.template | 2 +- machines/archer/jobscript-precompile-no-run.template | 2 +- .../archer/jobscript-precompile-plots-post-processing.template | 2 +- machines/archer/jobscript-precompile.template | 2 +- .../jobscript-precompile-makie-post-processing.template | 2 +- .../generic-batch-template/jobscript-precompile-no-run.template | 2 +- .../jobscript-precompile-plots-post-processing.template | 2 +- machines/generic-batch-template/jobscript-precompile.template | 2 +- .../marconi/jobscript-precompile-makie-post-processing.template | 2 +- machines/marconi/jobscript-precompile-no-run.template | 2 +- .../marconi/jobscript-precompile-plots-post-processing.template | 2 +- machines/marconi/jobscript-precompile.template | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/machines/archer/jobscript-precompile-makie-post-processing.template b/machines/archer/jobscript-precompile-makie-post-processing.template index 8e82da0c1..b3bd6fdf2 100644 --- a/machines/archer/jobscript-precompile-makie-post-processing.template +++ b/machines/archer/jobscript-precompile-makie-post-processing.template @@ -24,4 +24,4 @@ echo "precompiling post-processing $(date)" bin/julia --project=makie_post_processing/ precompile-makie-post-processing.jl -echo "finished!" +echo "finished! $(date)" diff --git a/machines/archer/jobscript-precompile-no-run.template b/machines/archer/jobscript-precompile-no-run.template index 3fc0465c5..0a800be4d 100644 --- a/machines/archer/jobscript-precompile-no-run.template +++ b/machines/archer/jobscript-precompile-no-run.template @@ -24,4 +24,4 @@ echo "precompiling $(date)" bin/julia --project -O3 --check-bounds=no precompile-no-run.jl -echo "finished!" +echo "finished! $(date)" diff --git a/machines/archer/jobscript-precompile-plots-post-processing.template b/machines/archer/jobscript-precompile-plots-post-processing.template index 543455a15..9f53698ee 100644 --- a/machines/archer/jobscript-precompile-plots-post-processing.template +++ b/machines/archer/jobscript-precompile-plots-post-processing.template @@ -24,4 +24,4 @@ echo "precompiling post-processing $(date)" bin/julia --project=plots_post_processing/ precompile-plots-post-processing.jl -echo "finished!" +echo "finished! $(date)" diff --git a/machines/archer/jobscript-precompile.template b/machines/archer/jobscript-precompile.template index f405df982..8213b7e01 100644 --- a/machines/archer/jobscript-precompile.template +++ b/machines/archer/jobscript-precompile.template @@ -24,4 +24,4 @@ echo "precompiling $(date)" bin/julia --project -O3 --check-bounds=no precompile.jl -echo "finished!" +echo "finished! $(date)" diff --git a/machines/generic-batch-template/jobscript-precompile-makie-post-processing.template b/machines/generic-batch-template/jobscript-precompile-makie-post-processing.template index c3db0bcfb..85c789562 100644 --- a/machines/generic-batch-template/jobscript-precompile-makie-post-processing.template +++ b/machines/generic-batch-template/jobscript-precompile-makie-post-processing.template @@ -17,4 +17,4 @@ echo "precompiling $(date)" bin/julia --project=makie_post_processing/ precompile-makie-post-processing.jl -echo "finished!" +echo "finished! $(date)" diff --git a/machines/generic-batch-template/jobscript-precompile-no-run.template b/machines/generic-batch-template/jobscript-precompile-no-run.template index 55571557b..af1b37369 100644 --- a/machines/generic-batch-template/jobscript-precompile-no-run.template +++ b/machines/generic-batch-template/jobscript-precompile-no-run.template @@ -17,4 +17,4 @@ echo "precompiling $(date)" bin/julia --project -O3 --check-bounds=no precompile-no-run.jl -echo "finished!" +echo "finished! $(date)" diff --git a/machines/generic-batch-template/jobscript-precompile-plots-post-processing.template b/machines/generic-batch-template/jobscript-precompile-plots-post-processing.template index b71f7e09d..ea81a39c9 100644 --- a/machines/generic-batch-template/jobscript-precompile-plots-post-processing.template +++ b/machines/generic-batch-template/jobscript-precompile-plots-post-processing.template @@ -17,4 +17,4 @@ echo "precompiling $(date)" bin/julia --project=plots_post_processing/ precompile-plots-post-processing.jl -echo "finished!" +echo "finished! $(date)" diff --git a/machines/generic-batch-template/jobscript-precompile.template b/machines/generic-batch-template/jobscript-precompile.template index fd26c88e4..212ac53fa 100644 --- a/machines/generic-batch-template/jobscript-precompile.template +++ b/machines/generic-batch-template/jobscript-precompile.template @@ -17,4 +17,4 @@ echo "precompiling $(date)" bin/julia --project -O3 --check-bounds=no precompile.jl -echo "finished!" +echo "finished! $(date)" diff --git a/machines/marconi/jobscript-precompile-makie-post-processing.template b/machines/marconi/jobscript-precompile-makie-post-processing.template index 73ce99ad0..ea0d7a6cb 100644 --- a/machines/marconi/jobscript-precompile-makie-post-processing.template +++ b/machines/marconi/jobscript-precompile-makie-post-processing.template @@ -17,4 +17,4 @@ echo "precompiling $(date)" bin/julia --project=makie_post_processing/ precompile-makie-post-processing.jl -echo "finished!" +echo "finished! $(date)" diff --git a/machines/marconi/jobscript-precompile-no-run.template b/machines/marconi/jobscript-precompile-no-run.template index e5bbb9cc9..241f20956 100644 --- a/machines/marconi/jobscript-precompile-no-run.template +++ b/machines/marconi/jobscript-precompile-no-run.template @@ -17,4 +17,4 @@ echo "precompiling $(date)" bin/julia --project -O3 --check-bounds=no precompile-no-run.jl -echo "finished!" +echo "finished! $(date)" diff --git a/machines/marconi/jobscript-precompile-plots-post-processing.template b/machines/marconi/jobscript-precompile-plots-post-processing.template index 60235ad78..4b0d7be07 100644 --- a/machines/marconi/jobscript-precompile-plots-post-processing.template +++ b/machines/marconi/jobscript-precompile-plots-post-processing.template @@ -17,4 +17,4 @@ echo "precompiling $(date)" bin/julia --project=plots_post_processing/ precompile-plots-post-processing.jl -echo "finished!" +echo "finished! $(date)" diff --git a/machines/marconi/jobscript-precompile.template b/machines/marconi/jobscript-precompile.template index 651b2fc92..d7528d70e 100644 --- a/machines/marconi/jobscript-precompile.template +++ b/machines/marconi/jobscript-precompile.template @@ -17,4 +17,4 @@ echo "precompiling $(date)" bin/julia --project -O3 --check-bounds=no precompile.jl -echo "finished!" +echo "finished! $(date)" From e9a9f5bb5d2449e3ed150c23a326c556d8d8af44 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 2 Jan 2024 17:28:46 +0000 Subject: [PATCH 61/80] Scripts to chain precompilation with simulation run/restart submission --- .gitignore | 2 + README.md | 24 +++++++++ machines/shared/machine_setup_stage_two.jl | 2 + ...precompile-makie-post-processing-submit.sh | 16 ++++-- ...precompile-plots-post-processing-submit.sh | 16 ++++-- machines/shared/precompile-submit.sh | 25 +++++++-- .../shared/submit-precompile-and-restart.sh | 54 +++++++++++++++++++ machines/shared/submit-precompile-and-run.sh | 54 +++++++++++++++++++ machines/shared/submit-restart.sh | 34 ++++++++---- machines/shared/submit-run.sh | 34 ++++++++---- 10 files changed, 232 insertions(+), 29 deletions(-) create mode 100755 machines/shared/submit-precompile-and-restart.sh create mode 100755 machines/shared/submit-precompile-and-run.sh diff --git a/.gitignore b/.gitignore index a99a993c3..4d3aedd9f 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ LocalPreferences.toml /precompile-submit.sh /submit-restart.sh /submit-run.sh +/submit-precompile-and-restart.sh +/submit-precompile-and-run.sh /.julia_default.txt /.julia_directory_default.txt /.this_machine_name.txt diff --git a/README.md b/README.md index 65ae6a46c..119667199 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,18 @@ $ ./submit-run.sh -h for various command line options to change parameters (e.g. number of nodes, etc.). +If you need to rebuild the system images `moment_kinetics.so` and +`makie_postproc.so` or `plots_postproc.so` because you have updated the code +since they were built, it may be convenient to use +``` +$ ./submit-precompile-and-run.sh input.toml +``` +which will submit jobs for compilation, to run the simulation, and to do post +processing. The simulation job will wait for the compilation job creating +`moment_kinetics.so` to finish before starting. The post processing job will +wait for the compilation job creating `makie_postproc.so` or +`plots_postproc.so` to finish before starting. + ### Stopping a run When running in the REPL (especially with MPI) interrupting a run using Ctrl-C @@ -234,6 +246,18 @@ $ ./submit-restart.sh -h for various other command line options to change parameters (e.g. number of nodes, etc.). +If you need to rebuild the system images `moment_kinetics.so` and +`makie_postproc.so` or `plots_postproc.so` because you have updated the code +since they were built, it may be convenient to use +``` +$ ./submit-precompile-and-restart.sh [-r runs/example/example.dfns.h5] input.toml +``` +which will submit jobs for compilation, to restart the simulation, and to do +post processing. The simulation job will wait for the compilation job creating +`moment_kinetics.so` to finish before starting. The post processing job will +wait for the compilation job creating `makie_postproc.so` or +`plots_postproc.so` to finish before starting. + It is possible to restart a run from another output file with different resolution settings or different moment-kinetic options. This is done by interpolating variables from the old run onto the new grid. diff --git a/machines/shared/machine_setup_stage_two.jl b/machines/shared/machine_setup_stage_two.jl index 2fba87d35..9228563bb 100644 --- a/machines/shared/machine_setup_stage_two.jl +++ b/machines/shared/machine_setup_stage_two.jl @@ -41,6 +41,8 @@ if batch_system make_batch_symlink("precompile-submit.sh") make_batch_symlink("submit-run.sh") make_batch_symlink("submit-restart.sh") + make_batch_symlink("submit-precompile-and-run.sh") + make_batch_symlink("submit-precompile-and-restart.sh") if mk_preferences["use_makie"] == "y" make_batch_symlink("precompile-makie-post-processing-submit.sh") end diff --git a/machines/shared/precompile-makie-post-processing-submit.sh b/machines/shared/precompile-makie-post-processing-submit.sh index 73a2a3fc0..3c0b839a2 100755 --- a/machines/shared/precompile-makie-post-processing-submit.sh +++ b/machines/shared/precompile-makie-post-processing-submit.sh @@ -3,13 +3,17 @@ set -e # Parse command line options -while getopts "h" opt; do +ONLY_JOB_ID=1 +while getopts "hj" opt; do case $opt in h) echo "Submit job to precompile moment kinetics -h Print help and exit" exit 1 ;; + j) + ONLY_JOB_ID=0 + ;; esac done @@ -31,7 +35,13 @@ POSTPROCESSINGJOBSCRIPT=${PRECOMPILEDIR}precompile-makie-post-processing.job sed -e "s|ACCOUNT|$ACCOUNT|" -e "s|PRECOMPILEDIR|$PRECOMPILEDIR|" machines/$MACHINE/jobscript-precompile-makie-post-processing.template > $POSTPROCESSINGJOBSCRIPT JOBID=$(sbatch --parsable $POSTPROCESSINGJOBSCRIPT) -echo "Precompile makie_post_processing: $JOBID" +if [[ "$ONLY_JOB_ID" -eq 1 ]]; then + echo "Precompile makie_post_processing: $JOBID" +else + echo "$JOBID" +fi echo "In the queue" > $PRECOMPILEDIR/slurm-$JOBID.out -echo "Done" +if [[ "$ONLY_JOB_ID" -eq 1 ]]; then + echo "Done" +fi diff --git a/machines/shared/precompile-plots-post-processing-submit.sh b/machines/shared/precompile-plots-post-processing-submit.sh index f5d7e6b45..5ae058486 100755 --- a/machines/shared/precompile-plots-post-processing-submit.sh +++ b/machines/shared/precompile-plots-post-processing-submit.sh @@ -3,13 +3,17 @@ set -e # Parse command line options -while getopts "h" opt; do +ONLY_JOB_ID=1 +while getopts "hj" opt; do case $opt in h) echo "Submit job to precompile moment kinetics -h Print help and exit" exit 1 ;; + j) + ONLY_JOB_ID=0 + ;; esac done @@ -31,7 +35,13 @@ POSTPROCESSINGJOBSCRIPT=${PRECOMPILEDIR}precompile-plots-post-processing.job sed -e "s|ACCOUNT|$ACCOUNT|" -e "s|PRECOMPILEDIR|$PRECOMPILEDIR|" machines/$MACHINE/jobscript-precompile-plots-post-processing.template > $POSTPROCESSINGJOBSCRIPT JOBID=$(sbatch --parsable $POSTPROCESSINGJOBSCRIPT) -echo "Precompile plots_post_processing: $JOBID" +if [[ "$ONLY_JOB_ID" -eq 1 ]]; then + echo "Precompile plots_post_processing: $JOBID" +else + echo "$JOBID" +fi echo "In the queue" > $PRECOMPILEDIR/slurm-$JOBID.out -echo "Done" +if [[ "$ONLY_JOB_ID" -eq 1 ]]; then + echo "Done" +fi diff --git a/machines/shared/precompile-submit.sh b/machines/shared/precompile-submit.sh index dd09b0bfb..c61843c5e 100755 --- a/machines/shared/precompile-submit.sh +++ b/machines/shared/precompile-submit.sh @@ -3,17 +3,22 @@ set -e # Parse command line options +ONLY_JOB_ID=1 NO_PRECOMPILE_RUN=1 NO_POSTPROC=1 -while getopts "hno" opt; do +while getopts "hjno" opt; do case $opt in h) echo "Submit job to precompile moment kinetics -h Print help and exit +-j Only print job id (so it can be captured easily by another script) -n No 'precompile run' - i.e. call precompile-no-run.jl instead of precompile.jl -o Only compile moment_kinetics.so, skipping any available post-processing packages" exit 1 ;; + j) + ONLY_JOB_ID=0 + ;; n) NO_PRECOMPILE_RUN=0 ;; @@ -40,16 +45,24 @@ mkdir -p $PRECOMPILEDIR # Create a submission script for the precompilation JOBSCRIPT=${PRECOMPILEDIR}precompile.job if [[ NO_PRECOMPILE_RUN -eq 0 ]]; then - echo "Submitting precompile job (no precompile run)..." + if [[ "$ONLY_JOB_ID" -eq 1 ]]; then + echo "Submitting precompile job (no precompile run)..." + fi TEMPLATE_NAME=jobscript-precompile-no-run.template else - echo "Submitting precompile job..." + if [[ "$ONLY_JOB_ID" -eq 1 ]]; then + echo "Submitting precompile job..." + fi TEMPLATE_NAME=jobscript-precompile.template fi sed -e "s|ACCOUNT|$ACCOUNT|" -e "s|PRECOMPILEDIR|$PRECOMPILEDIR|" machines/$MACHINE/$TEMPLATE_NAME > $JOBSCRIPT JOBID=$(sbatch --parsable $JOBSCRIPT) -echo "Precompile: $JOBID" +if [[ "$ONLY_JOB_ID" -eq 1 ]]; then + echo "Precompile: $JOBID" +else + echo "$JOBID" +fi echo "In the queue" > $PRECOMPILEDIR/slurm-$JOBID.out if [[ "$NO_POSTPROC" -eq 1 && "$MAKIE_AVAILABLE" == "y" ]]; then @@ -59,4 +72,6 @@ if [[ "$NO_POSTPROC" -eq 1 && "$PLOTS_AVAILABLE" == "y" ]]; then ./precompile-plots-post-processing-submit.sh fi -echo "Done" +if [[ "$ONLY_JOB_ID" -eq 1 ]]; then + echo "Done" +fi diff --git a/machines/shared/submit-precompile-and-restart.sh b/machines/shared/submit-precompile-and-restart.sh new file mode 100755 index 000000000..554638c7b --- /dev/null +++ b/machines/shared/submit-precompile-and-restart.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +set -e + +POSTPROC=0 +MAKIEPOSTPROCESS=1 +while getopts "haf:g:m:n:p:oq:r:st:u:w" opt; do + case $opt in + h) + echo "Precompile the system image(s) and then restart a simulation. Arguments are passed through to submit-restart.sh:" + echo + ./submit-restart.sh -h + exit 1 + ;; + a) + POSTPROC=1 + ;; + o) + MAKIEPOSTPROCESS=0 + ;; + esac +done + +source julia.env + +JOBINFO=($(util/get-precompile-info.jl)) +MACHINE=${JOBINFO[0]} +ACCOUNT=${JOBINFO[1]} +MAKIE_AVAILABLE=${JOBINFO[2]} +PLOTS_AVAILABLE=${JOBINFO[3]} + +if [[ "$MAKIE_AVAILABLE" == "n" && "$PLOTS_AVAILABLE" == "y" ]]; then + # No Makie post-processing available, so always use Plots post-processing + MAKIEPOSTPROCESS=0 +fi + +MK_SYSIMAGE_JOB_ID=$(./precompile-submit.sh -o -j) +echo "Precompile moment_kinetics: $MK_SYSIMAGE_JOB_ID" + +POSTPROC_ARGUMENT="" +if [[ $POSTPROC -eq 0 && ("$MAKIE_AVAILABLE" == "y" || "$PLOTS_AVAILABLE" == "y") ]]; then + # Create a submission script for post-processing + if [[ MAKIEPOSTPROCESS -eq 1 ]]; then + POSTPROC_SYSIMAGE_JOB_ID=$(./precompile-makie-post-processing-submit.sh -j) + echo "Precompile makie_post_processing: $POSTPROC_SYSIMAGE_JOB_ID" + else + POSTPROC_SYSIMAGE_JOB_ID=$(./precompile-plots-post-processing-submit.sh -j) + echo "Precompile plots_post_processing: $POSTPROC_SYSIMAGE_JOB_ID" + fi + + POSTPROC_ARGUMENT="-g $POSTPROC_SYSIMAGE_JOB_ID" +fi + +./submit-restart.sh -f $MK_SYSIMAGE_JOB_ID $POSTPROC_ARGUMENT -w $@ diff --git a/machines/shared/submit-precompile-and-run.sh b/machines/shared/submit-precompile-and-run.sh new file mode 100755 index 000000000..f53a58326 --- /dev/null +++ b/machines/shared/submit-precompile-and-run.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +set -e + +POSTPROC=0 +MAKIEPOSTPROCESS=1 +while getopts "haf:g:m:n:p:oq:st:u:w" opt; do + case $opt in + h) + echo "Precompile the system image(s) and then run a simulation. Arguments are passed through to submit-run.sh:" + echo + ./submit-run.sh -h + exit 1 + ;; + a) + POSTPROC=1 + ;; + o) + MAKIEPOSTPROCESS=0 + ;; + esac +done + +source julia.env + +JOBINFO=($(util/get-precompile-info.jl)) +MACHINE=${JOBINFO[0]} +ACCOUNT=${JOBINFO[1]} +MAKIE_AVAILABLE=${JOBINFO[2]} +PLOTS_AVAILABLE=${JOBINFO[3]} + +if [[ "$MAKIE_AVAILABLE" == "n" && "$PLOTS_AVAILABLE" == "y" ]]; then + # No Makie post-processing available, so always use Plots post-processing + MAKIEPOSTPROCESS=0 +fi + +MK_SYSIMAGE_JOB_ID=$(./precompile-submit.sh -o -j) +echo "Precompile moment_kinetics: $MK_SYSIMAGE_JOB_ID" + +POSTPROC_ARGUMENT="" +if [[ $POSTPROC -eq 0 && ("$MAKIE_AVAILABLE" == "y" || "$PLOTS_AVAILABLE" == "y") ]]; then + # Create a submission script for post-processing + if [[ MAKIEPOSTPROCESS -eq 1 ]]; then + POSTPROC_SYSIMAGE_JOB_ID=$(./precompile-makie-post-processing-submit.sh -j) + echo "Precompile makie_post_processing: $POSTPROC_SYSIMAGE_JOB_ID" + else + POSTPROC_SYSIMAGE_JOB_ID=$(./precompile-plots-post-processing-submit.sh -j) + echo "Precompile plots_post_processing: $POSTPROC_SYSIMAGE_JOB_ID" + fi + + POSTPROC_ARGUMENT="-g $POSTPROC_SYSIMAGE_JOB_ID" +fi + +./submit-run.sh -f $MK_SYSIMAGE_JOB_ID $POSTPROC_ARGUMENT -w $@ diff --git a/machines/shared/submit-restart.sh b/machines/shared/submit-restart.sh index b4bedec51..46a30a2f1 100755 --- a/machines/shared/submit-restart.sh +++ b/machines/shared/submit-restart.sh @@ -27,8 +27,10 @@ POSTPROC=0 SUBMIT=0 RESTARTFROM="" FOLLOWFROM="" +POSTPROC_FOLLOWFROM="" MAKIEPOSTPROCESS=1 -while getopts "haf:m:n:p:oq:r:st:u:" opt; do +WARN_OLD_SYSIMAGE=0 +while getopts "haf:g:m:n:p:oq:r:st:u:w" opt; do case $opt in h) echo "Submit jobs for a simulation (using INPUT_FILE for input) and post-processing to the queue @@ -36,6 +38,7 @@ Usage: submit-run.sh [option] INPUT_FILE -h Print help and exit -a Do not submit post-processing job after the run -f JOBID Make this job start after JOBID finishes successfully +-g JOBID Make the post processing job start after JOBID finishes successfully (as well as after the simulation run finishes) -m MEM The requested memory for post-processing -n NODES The number of nodes to use for the simulation -o Use original post_processing, instead of makie_post_processing, for the post-processing job when both are available @@ -44,7 +47,8 @@ Usage: submit-run.sh [option] INPUT_FILE -r FILE The output file to restart from (defaults to latest output in the run directory) -s Only create submission scripts, do not actually submit jobs -t TIME The run time, e.g. 24:00:00 --u TIME The run time for the post-processing, e.g. 1:00:00" +-u TIME The run time for the post-processing, e.g. 1:00:00 +-w Suppress the warning given when system image(s) are older than source code files" exit 1 ;; a) @@ -53,6 +57,9 @@ Usage: submit-run.sh [option] INPUT_FILE f) FOLLOWFROM="-d afterok:$OPTARG" ;; + g) + POSTPROC_FOLLOWFROM=",afterok:$OPTARG" + ;; m) POSTPROCMEM=$OPTARG ;; @@ -80,6 +87,9 @@ Usage: submit-run.sh [option] INPUT_FILE u) POSTPROCTIME=$OPTARG ;; + w) + WARN_OLD_SYSIMAGE=1 + ;; esac done @@ -111,8 +121,10 @@ fi RESTARTJOBSCRIPT=${RUNDIR}$RUNNAME-restart.job sed -e "s|NODES|$NODES|" -e "s|RUNTIME|$RUNTIME|" -e "s|ACCOUNT|$ACCOUNT|" -e "s|PARTITION|$PARTITION|" -e "s|QOS|$QOS|" -e "s|RUNDIR|$RUNDIR|" -e "s|INPUTFILE|$INPUTFILE|" -e "s|RESTARTFROM|$RESTARTFROM|" machines/$MACHINE/jobscript-restart.template > $RESTARTJOBSCRIPT -# Check that source code has not been changed since moment_kinetics.so was created -bin/julia --project -O3 --check-bounds=no util/check_so_newer_than_code.jl moment_kinetics.so +if [[ "$WARN_OLD_SYSIMAGE" -eq 0 ]]; then + # Check that source code has not been changed since moment_kinetics.so was created + bin/julia --project -O3 --check-bounds=no util/check_so_newer_than_code.jl moment_kinetics.so +fi if [[ $SUBMIT -eq 0 ]]; then JOBID=$(sbatch $FOLLOWFROM --parsable $RESTARTJOBSCRIPT) @@ -128,18 +140,22 @@ elif [[ $POSTPROC -eq 0 ]]; then if [[ MAKIEPOSTPROCESS -eq 1 ]]; then POSTPROCESSTEMPLATE=jobscript-postprocess.template - # Check that source code has not been changed since makie_postproc.so was created - bin/julia --project=makie_post_processing -O3 util/check_so_newer_than_code.jl makie_postproc.so + if [[ "$WARN_OLD_SYSIMAGE" -eq 0 ]]; then + # Check that source code has not been changed since makie_postproc.so was created + bin/julia --project=makie_post_processing -O3 util/check_so_newer_than_code.jl makie_postproc.so + fi else POSTPROCESSTEMPLATE=jobscript-postprocess-plotsjl.template - # Check that source code has not been changed since plots_postproc.so was created - bin/julia --project=plots_post_processing -O3 util/check_so_newer_than_code.jl plots_postproc.so + if [[ "$WARN_OLD_SYSIMAGE" -eq 0 ]]; then + # Check that source code has not been changed since plots_postproc.so was created + bin/julia --project=plots_post_processing -O3 util/check_so_newer_than_code.jl plots_postproc.so + fi fi sed -e "s|POSTPROCMEMORY|$POSTPROCMEMORY|" -e "s|POSTPROCTIME|$POSTPROCTIME|" -e "s|ACCOUNT|$ACCOUNT|" -e "s|RUNDIR|$RUNDIR|" machines/$MACHINE/$POSTPROCESSTEMPLATE > $POSTPROCJOBSCRIPT if [[ $SUBMIT -eq 0 ]]; then - POSTID=$(sbatch -d afterany:$JOBID --parsable $POSTPROCJOBSCRIPT) + POSTID=$(sbatch -d afterany:$JOBID$POSTPROC_FOLLOWFROM --parsable $POSTPROCJOBSCRIPT) echo "Postprocess: $POSTID" echo "In the queue" > ${RUNDIR}slurm-post-$POSTID.out fi diff --git a/machines/shared/submit-run.sh b/machines/shared/submit-run.sh index ee363dd99..38a9639f4 100755 --- a/machines/shared/submit-run.sh +++ b/machines/shared/submit-run.sh @@ -26,8 +26,10 @@ PLOTS_AVAILABLE=${JOBINFO[9]} POSTPROC=0 SUBMIT=0 FOLLOWFROM="" +POSTPROC_FOLLOWFROM="" MAKIEPOSTPROCESS=1 -while getopts "haf:m:n:p:oq:st:u:" opt; do +WARN_OLD_SYSIMAGE=0 +while getopts "haf:g:m:n:p:oq:st:u:w" opt; do case $opt in h) echo "Submit jobs for a simulation (using INPUT_FILE for input) and post-processing to the queue @@ -35,6 +37,7 @@ Usage: submit-run.sh [option] INPUT_FILE -h Print help and exit -a Do not submit post-processing job after the run -f JOBID Make this job start after JOBID finishes successfully +-g JOBID Make the post processing job start after JOBID finishes successfully (as well as after the simulation run finishes) -m MEM The requested memory for post-processing -n NODES The number of nodes to use for the simulation -o Use original post_processing, instead of makie_post_processing, for the post-processing job when both are available @@ -42,7 +45,8 @@ Usage: submit-run.sh [option] INPUT_FILE -q QOS The 'quality of service' (passed to 'sbatch --qos') -s Only create submission scripts, do not actually submit jobs -t TIME The run time, e.g. 24:00:00 --u TIME The run time for the post-processing, e.g. 1:00:00" +-u TIME The run time for the post-processing, e.g. 1:00:00 +-w Suppress the warning given when system image(s) are older than source code files" exit 1 ;; a) @@ -51,6 +55,9 @@ Usage: submit-run.sh [option] INPUT_FILE f) FOLLOWFROM="-d afterok:$OPTARG" ;; + g) + POSTPROC_FOLLOWFROM=",afterok:$OPTARG" + ;; m) POSTPROCMEM=$OPTARG ;; @@ -75,6 +82,9 @@ Usage: submit-run.sh [option] INPUT_FILE u) POSTPROCTIME=$OPTARG ;; + w) + WARN_OLD_SYSIMAGE=1 + ;; esac done @@ -106,8 +116,10 @@ mkdir -p $RUNDIR RUNJOBSCRIPT=${RUNDIR}$RUNNAME.job sed -e "s|NODES|$NODES|" -e "s|RUNTIME|$RUNTIME|" -e "s|ACCOUNT|$ACCOUNT|" -e "s|PARTITION|$PARTITION|" -e "s|QOS|$QOS|" -e "s|RUNDIR|$RUNDIR|" -e "s|INPUTFILE|$INPUTFILE|" machines/$MACHINE/jobscript-run.template > $RUNJOBSCRIPT -# Check that source code has not been changed since moment_kinetics.so was created -bin/julia --project -O3 --check-bounds=no util/check_so_newer_than_code.jl moment_kinetics.so +if [[ "$WARN_OLD_SYSIMAGE" -eq 0 ]]; then + # Check that source code has not been changed since moment_kinetics.so was created + bin/julia --project -O3 --check-bounds=no util/check_so_newer_than_code.jl moment_kinetics.so +fi if [[ $SUBMIT -eq 0 ]]; then JOBID=$(sbatch $FOLLOWFROM --parsable $RUNJOBSCRIPT) @@ -123,18 +135,22 @@ elif [[ $POSTPROC -eq 0 ]]; then if [[ MAKIEPOSTPROCESS -eq 1 ]]; then POSTPROCESSTEMPLATE=jobscript-postprocess.template - # Check that source code has not been changed since makie_postproc.so was created - bin/julia --project=makie_post_processing -O3 util/check_so_newer_than_code.jl makie_postproc.so + if [[ "$WARN_OLD_SYSIMAGE" -eq 0 ]]; then + # Check that source code has not been changed since makie_postproc.so was created + bin/julia --project=makie_post_processing -O3 util/check_so_newer_than_code.jl makie_postproc.so + fi else POSTPROCESSTEMPLATE=jobscript-postprocess-plotsjl.template - # Check that source code has not been changed since plots_postproc.so was created - bin/julia --project=plots_post_processing -O3 util/check_so_newer_than_code.jl plots_postproc.so + if [[ "$WARN_OLD_SYSIMAGE" -eq 0 ]]; then + # Check that source code has not been changed since plots_postproc.so was created + bin/julia --project=plots_post_processing -O3 util/check_so_newer_than_code.jl plots_postproc.so + fi fi sed -e "s|POSTPROCMEMORY|$POSTPROCMEMORY|" -e "s|POSTPROCTIME|$POSTPROCTIME|" -e "s|ACCOUNT|$ACCOUNT|" -e "s|RUNDIR|$RUNDIR|" machines/$MACHINE/$POSTPROCESSTEMPLATE > $POSTPROCJOBSCRIPT if [[ $SUBMIT -eq 0 ]]; then - POSTID=$(sbatch -d afterany:$JOBID --parsable $POSTPROCJOBSCRIPT) + POSTID=$(sbatch -d afterany:$JOBID$POSTPROC_FOLLOWFROM --parsable $POSTPROCJOBSCRIPT) echo "Postprocess: $POSTID" echo "In the queue" > ${RUNDIR}slurm-post-$POSTID.out fi From bdc5362c7185c5b5a7fccc0a576b329c9dd3d765 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 2 Jan 2024 17:51:46 +0000 Subject: [PATCH 62/80] Remove no-longer-necessary force exit from machine_setup_stage_two.jl The MPI and HDF5 setup is no longer done in the machine_setup_stage_two.jl script. It is also not really expected that the script will be run other than from machine_setup.sh. So the message saying Julia is exiting and the forced exit by calling `exit(0)` are not really necessary. --- machines/shared/machine_setup_stage_two.jl | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/machines/shared/machine_setup_stage_two.jl b/machines/shared/machine_setup_stage_two.jl index 9228563bb..5c8714d1e 100644 --- a/machines/shared/machine_setup_stage_two.jl +++ b/machines/shared/machine_setup_stage_two.jl @@ -54,13 +54,3 @@ if batch_system run(`./precompile-submit.sh -o`) end end - - -# Force exit so Julia must be restarted -####################################### - -println() -println("************************************************************") -println("Julia must be restarted to use the updated MPI, exiting now.") -println("************************************************************") -exit(0) From 59076dee9bb4110be0d1a49b4b1ca2713ed9a51e Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 2 Jan 2024 20:07:44 +0000 Subject: [PATCH 63/80] Use julia-1.10 in CI Thanks to fixes that mean julia-1.10 should work better with MPI, it is now the recommended version, so use julia-1.10 for the CI automated tests. --- .github/workflows/debug_checks.yml | 2 +- .github/workflows/documentation.yml | 2 +- .github/workflows/examples.yml | 2 +- .github/workflows/longtest.yml | 2 +- .github/workflows/parallel_test.yml | 4 ++-- .github/workflows/test.yml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/debug_checks.yml b/.github/workflows/debug_checks.yml index a6113e7ec..94c0b864a 100644 --- a/.github/workflows/debug_checks.yml +++ b/.github/workflows/debug_checks.yml @@ -20,7 +20,7 @@ jobs: mpi: 'openmpi' - uses: julia-actions/setup-julia@v1 with: - version: '1.9' + version: '1.10' arch: x64 - uses: julia-actions/julia-buildpkg@v1 with: diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index c1117cf02..b9a550c24 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: '1.8' + version: '1.10' - name: Install dependencies run: | pip3 install --user matplotlib diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index e3afd3099..17b932a5d 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: - version: '1.9' + version: '1.10' arch: x64 - name: Test examples run: | diff --git a/.github/workflows/longtest.yml b/.github/workflows/longtest.yml index 9c679c2c8..55e6ffaaf 100644 --- a/.github/workflows/longtest.yml +++ b/.github/workflows/longtest.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: - version: '1.9' + version: '1.10' arch: x64 - uses: julia-actions/julia-buildpkg@v1 with: diff --git a/.github/workflows/parallel_test.yml b/.github/workflows/parallel_test.yml index b2734b1db..e569fe0e3 100644 --- a/.github/workflows/parallel_test.yml +++ b/.github/workflows/parallel_test.yml @@ -15,7 +15,7 @@ jobs: mpi: 'openmpi' - uses: julia-actions/setup-julia@v1 with: - version: '1.9' + version: '1.10' arch: x64 - run: | touch Project.toml @@ -45,7 +45,7 @@ jobs: mpi: 'openmpi' - uses: julia-actions/setup-julia@v1 with: - version: '1.9' + version: '1.10' arch: x64 - run: | touch Project.toml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ff5a21ae1..138295163 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: - version: '1.9' + version: '1.10' arch: x64 - uses: julia-actions/julia-buildpkg@v1 with: From c3ad03da687718d1ef6a6953a728598682e08a41 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 2 Jan 2024 20:40:44 +0000 Subject: [PATCH 64/80] Fix documentation build CI job --- .github/workflows/documentation.yml | 1 - docs/make.jl | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index b9a550c24..82219cc48 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -18,7 +18,6 @@ jobs: - name: Install dependencies run: | pip3 install --user matplotlib - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=joinpath(pwd(), moment_kinetics))); Pkg.develop(PackageSpec(path=joinpath(pwd(), "makie_post_processing"))); Pkg.develop(PackageSpec(path=joinpath(pwd(), "plots_post_processing"))); Pkg.instantiate()' - name: Build and deploy env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Authenticate with GitHub Actions token diff --git a/docs/make.jl b/docs/make.jl index 5a5130564..18aa253e6 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,3 +1,11 @@ +using Pkg + +repo_dir = dirname(dirname(@__FILE__)) +Pkg.develop([PackageSpec(path=joinpath(repo_dir, "moment_kinetics")), + PackageSpec(path=joinpath(repo_dir, "makie_post_processing", "makie_post_processing")), + PackageSpec(path=joinpath(repo_dir, "plots_post_processing", "plots_post_processing"))]) +Pkg.instantiate() + using Documenter using moment_kinetics, makie_post_processing, plots_post_processing From 02e56ab7313e3a63836a34af7e1921ced94bb128 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 3 Jan 2024 12:19:48 +0000 Subject: [PATCH 65/80] Relax tolerance slightly in calculus_tests Rounding error sometimes causes chebyshev_pseudospectral matrix-multiplication derivative test to fail on macOS, so use 2e15 instead of 1e15 for the tolerance. --- moment_kinetics/test/calculus_tests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moment_kinetics/test/calculus_tests.jl b/moment_kinetics/test/calculus_tests.jl index c46c93af7..be8ac68ac 100644 --- a/moment_kinetics/test/calculus_tests.jl +++ b/moment_kinetics/test/calculus_tests.jl @@ -14,7 +14,7 @@ function runtests() println("calculus tests") @testset "fundamental theorem of calculus" begin @testset "$discretization $ngrid $nelement" for - (discretization, element_spacing_option, etol, cheb_option) ∈ (("finite_difference", "uniform", 1.0e-15, ""), ("chebyshev_pseudospectral", "uniform", 1.0e-15, "FFT"), ("chebyshev_pseudospectral", "uniform", 1.0e-15, "matrix"), ("chebyshev_pseudospectral", "sqrt", 1.0e-2, "FFT"), ("gausslegendre_pseudospectral", "uniform", 1.0e-14, "")), + (discretization, element_spacing_option, etol, cheb_option) ∈ (("finite_difference", "uniform", 1.0e-15, ""), ("chebyshev_pseudospectral", "uniform", 1.0e-15, "FFT"), ("chebyshev_pseudospectral", "uniform", 2.0e-15, "matrix"), ("chebyshev_pseudospectral", "sqrt", 1.0e-2, "FFT"), ("gausslegendre_pseudospectral", "uniform", 1.0e-14, "")), ngrid ∈ (5,6,7,8,9,10), nelement ∈ (1, 2, 3, 4, 5) if discretization == "finite_difference" && (ngrid - 1) * nelement % 2 == 1 From 3cc71ddb8c7c4ab94d67b9bdc6cc4889a2b22f20 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 3 Jan 2024 11:15:54 +0000 Subject: [PATCH 66/80] Move check_so_newer_than_code.jl to moment_kinetics/src/ This will allow the function to be used in moment_kinetics, makie_post_processing, or plots_post_processing. --- machines/shared/submit-restart.sh | 6 ++--- machines/shared/submit-run.sh | 6 ++--- .../src}/check_so_newer_than_code.jl | 23 +++++++++++++++---- moment_kinetics/src/moment_kinetics.jl | 1 + 4 files changed, 26 insertions(+), 10 deletions(-) rename {util => moment_kinetics/src}/check_so_newer_than_code.jl (64%) diff --git a/machines/shared/submit-restart.sh b/machines/shared/submit-restart.sh index 46a30a2f1..5589391ce 100755 --- a/machines/shared/submit-restart.sh +++ b/machines/shared/submit-restart.sh @@ -123,7 +123,7 @@ sed -e "s|NODES|$NODES|" -e "s|RUNTIME|$RUNTIME|" -e "s|ACCOUNT|$ACCOUNT|" -e "s if [[ "$WARN_OLD_SYSIMAGE" -eq 0 ]]; then # Check that source code has not been changed since moment_kinetics.so was created - bin/julia --project -O3 --check-bounds=no util/check_so_newer_than_code.jl moment_kinetics.so + bin/julia --project -O3 --check-bounds=no moment_kinetics/src/check_so_newer_than_code.jl moment_kinetics.so fi if [[ $SUBMIT -eq 0 ]]; then @@ -142,14 +142,14 @@ elif [[ $POSTPROC -eq 0 ]]; then if [[ "$WARN_OLD_SYSIMAGE" -eq 0 ]]; then # Check that source code has not been changed since makie_postproc.so was created - bin/julia --project=makie_post_processing -O3 util/check_so_newer_than_code.jl makie_postproc.so + bin/julia --project=makie_post_processing -O3 moment_kinetics/src/check_so_newer_than_code.jl makie_postproc.so fi else POSTPROCESSTEMPLATE=jobscript-postprocess-plotsjl.template if [[ "$WARN_OLD_SYSIMAGE" -eq 0 ]]; then # Check that source code has not been changed since plots_postproc.so was created - bin/julia --project=plots_post_processing -O3 util/check_so_newer_than_code.jl plots_postproc.so + bin/julia --project=plots_post_processing -O3 moment_kinetics/src/check_so_newer_than_code.jl plots_postproc.so fi fi sed -e "s|POSTPROCMEMORY|$POSTPROCMEMORY|" -e "s|POSTPROCTIME|$POSTPROCTIME|" -e "s|ACCOUNT|$ACCOUNT|" -e "s|RUNDIR|$RUNDIR|" machines/$MACHINE/$POSTPROCESSTEMPLATE > $POSTPROCJOBSCRIPT diff --git a/machines/shared/submit-run.sh b/machines/shared/submit-run.sh index 38a9639f4..b39efd4f6 100755 --- a/machines/shared/submit-run.sh +++ b/machines/shared/submit-run.sh @@ -118,7 +118,7 @@ sed -e "s|NODES|$NODES|" -e "s|RUNTIME|$RUNTIME|" -e "s|ACCOUNT|$ACCOUNT|" -e "s if [[ "$WARN_OLD_SYSIMAGE" -eq 0 ]]; then # Check that source code has not been changed since moment_kinetics.so was created - bin/julia --project -O3 --check-bounds=no util/check_so_newer_than_code.jl moment_kinetics.so + bin/julia --project -O3 --check-bounds=no moment_kinetics/src/check_so_newer_than_code.jl moment_kinetics.so fi if [[ $SUBMIT -eq 0 ]]; then @@ -137,14 +137,14 @@ elif [[ $POSTPROC -eq 0 ]]; then if [[ "$WARN_OLD_SYSIMAGE" -eq 0 ]]; then # Check that source code has not been changed since makie_postproc.so was created - bin/julia --project=makie_post_processing -O3 util/check_so_newer_than_code.jl makie_postproc.so + bin/julia --project=makie_post_processing -O3 moment_kinetics/src/check_so_newer_than_code.jl makie_postproc.so fi else POSTPROCESSTEMPLATE=jobscript-postprocess-plotsjl.template if [[ "$WARN_OLD_SYSIMAGE" -eq 0 ]]; then # Check that source code has not been changed since plots_postproc.so was created - bin/julia --project=plots_post_processing -O3 util/check_so_newer_than_code.jl plots_postproc.so + bin/julia --project=plots_post_processing -O3 moment_kinetics/src/check_so_newer_than_code.jl plots_postproc.so fi fi sed -e "s|POSTPROCMEMORY|$POSTPROCMEMORY|" -e "s|POSTPROCTIME|$POSTPROCTIME|" -e "s|ACCOUNT|$ACCOUNT|" -e "s|RUNDIR|$RUNDIR|" machines/$MACHINE/$POSTPROCESSTEMPLATE > $POSTPROCJOBSCRIPT diff --git a/util/check_so_newer_than_code.jl b/moment_kinetics/src/check_so_newer_than_code.jl similarity index 64% rename from util/check_so_newer_than_code.jl rename to moment_kinetics/src/check_so_newer_than_code.jl index 714711f16..0aef170a9 100644 --- a/util/check_so_newer_than_code.jl +++ b/moment_kinetics/src/check_so_newer_than_code.jl @@ -1,4 +1,17 @@ -function check_so_newer_than_code(so_filename = "moment_kinetics.so") +""" + check_so_newer_than_code(so_filename="moment_kinetics.so") + +Utility function that checks if `so_filename` is newer than the source code in +`moment_kinetics/src`. If it is, prints an error message and returns `false`; otherwise +returns `true`. + +If "makie" is found in `so_filename`, also checks against the source code in +`makie_post_processing/makie_post_processing/src/`. + +If "plots" is found in `so_filename`, also checks against the source code in +`plots_post_processing/plots_post_processing/src/`. +""" +function check_so_newer_than_code(so_filename="moment_kinetics.so") is_makie = occursin("makie", so_filename) is_plots = occursin("plots", so_filename) @@ -10,6 +23,8 @@ function check_so_newer_than_code(so_filename = "moment_kinetics.so") # Get modification time of *.so system image so_mtime = mtime(so_filename) + repo_dir = dirname(dirname(dirname(@__FILE__))) + # Get newest modification time of julia source file newest_jl_mtime = 0.0 @@ -21,12 +36,12 @@ function check_so_newer_than_code(so_filename = "moment_kinetics.so") end end end - get_newest_jl_mtime("moment_kinetics/src/") + get_newest_jl_mtime(joinpath(repo_dir, "moment_kinetics/src/")) if is_makie - get_newest_jl_mtime("makie_post_processing/makie_post_processing/src/") + get_newest_jl_mtime(joinpath(repo_dir, "makie_post_processing/makie_post_processing/src/")) end if is_plots - get_newest_jl_mtime("plots_post_processing/plots_post_processing/src/") + get_newest_jl_mtime(joinpath(repo_dir, "plots_post_processing/plots_post_processing/src/")) end so_is_newer = so_mtime > newest_jl_mtime diff --git a/moment_kinetics/src/moment_kinetics.jl b/moment_kinetics/src/moment_kinetics.jl index 70bdd4c3b..7c580d441 100644 --- a/moment_kinetics/src/moment_kinetics.jl +++ b/moment_kinetics/src/moment_kinetics.jl @@ -10,6 +10,7 @@ using MPI # Note that order of includes matters - things used in one module must already # be defined include("../../machines/shared/machine_setup.jl") # Included so Documenter.jl can find its docs +include("check_so_newer_than_code.jl") include("command_line_options.jl") include("constants.jl") include("debugging.jl") From 0af611843b6b70f33fca89490553842920735c45 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 3 Jan 2024 11:45:25 +0000 Subject: [PATCH 67/80] Optimise check_so_newer_than_code() If a source code file is found that is newer than the system image file, can stop checking. --- .../src/check_so_newer_than_code.jl | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/moment_kinetics/src/check_so_newer_than_code.jl b/moment_kinetics/src/check_so_newer_than_code.jl index 0aef170a9..6eb3f9972 100644 --- a/moment_kinetics/src/check_so_newer_than_code.jl +++ b/moment_kinetics/src/check_so_newer_than_code.jl @@ -26,26 +26,35 @@ function check_so_newer_than_code(so_filename="moment_kinetics.so") repo_dir = dirname(dirname(dirname(@__FILE__))) # Get newest modification time of julia source file - newest_jl_mtime = 0.0 + so_is_newer = true - function get_newest_jl_mtime(directory) + function check_file_mtimes(directory) for (root, dirs, files) ∈ walkdir(directory; follow_symlinks=true) for f ∈ files mt = mtime(joinpath(root, f)) - newest_jl_mtime = max(mt, newest_jl_mtime) + if mt > so_mtime + # Found a file that is newer than the .so + so_is_newer = false + break + end + end + if !so_is_newer + # Already found a file newer than the .so. No need to continue searching. + break end end end - get_newest_jl_mtime(joinpath(repo_dir, "moment_kinetics/src/")) - if is_makie - get_newest_jl_mtime(joinpath(repo_dir, "makie_post_processing/makie_post_processing/src/")) + check_file_mtimes(joinpath(repo_dir, "moment_kinetics/src/")) + + # If we already found a code file newer than the .so, no need to keep checking more + # files, so only keep checking if so_is_newer=true. + if so_is_newer && is_makie + check_file_mtimes(joinpath(repo_dir, "makie_post_processing/makie_post_processing/src/")) end - if is_plots - get_newest_jl_mtime(joinpath(repo_dir, "plots_post_processing/plots_post_processing/src/")) + if so_is_newer && is_plots + check_file_mtimes(joinpath(repo_dir, "plots_post_processing/plots_post_processing/src/")) end - so_is_newer = so_mtime > newest_jl_mtime - if !so_is_newer error_message = "WARNING: source code files have been modified more recently than\n" * From 0de8dde5b645c16356a6c0b3981edfa36d073573 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 3 Jan 2024 11:46:21 +0000 Subject: [PATCH 68/80] Check system image newer than source code when running moment_kinetics ...and when running post processing. --- .../src/makie_post_processing.jl | 9 +++++ .../src/check_so_newer_than_code.jl | 36 ++++++++++++++----- moment_kinetics/src/moment_kinetics.jl | 10 ++++++ .../src/plots_post_processing.jl | 9 +++++ 4 files changed, 55 insertions(+), 9 deletions(-) diff --git a/makie_post_processing/makie_post_processing/src/makie_post_processing.jl b/makie_post_processing/makie_post_processing/src/makie_post_processing.jl index 54fa303fb..1c0ff8859 100644 --- a/makie_post_processing/makie_post_processing/src/makie_post_processing.jl +++ b/makie_post_processing/makie_post_processing/src/makie_post_processing.jl @@ -21,6 +21,7 @@ include("shared_utils.jl") # Need this import just to allow links in the docstrings to be understood by Documenter.jl import moment_kinetics +using moment_kinetics: check_so_newer_than_code using moment_kinetics.analysis: analyze_fields_data, check_Chodura_condition, get_r_perturbation, get_Fourier_modes_2D, get_Fourier_modes_1D, steady_state_residuals, @@ -461,6 +462,14 @@ end function setup_makie_post_processing_input!(new_input_dict::AbstractDict{String,Any}; run_info_moments=nothing, run_info_dfns=nothing) + + # Check that, if we are using a custom compiled system image that includes + # moment_kinetics, the system image is newer than the source code files (if there are + # changes made to the source code since the system image was compiled, they will not + # affect the current run). Prints a warning if any code files are newer than the + # system image. + check_so_newer_than_code() + convert_to_OrderedDicts!(new_input_dict) if isa(run_info_moments, Tuple) diff --git a/moment_kinetics/src/check_so_newer_than_code.jl b/moment_kinetics/src/check_so_newer_than_code.jl index 6eb3f9972..a8154380f 100644 --- a/moment_kinetics/src/check_so_newer_than_code.jl +++ b/moment_kinetics/src/check_so_newer_than_code.jl @@ -1,19 +1,34 @@ """ - check_so_newer_than_code(so_filename="moment_kinetics.so") + check_so_newer_than_code(so_filename=nothing) Utility function that checks if `so_filename` is newer than the source code in `moment_kinetics/src`. If it is, prints an error message and returns `false`; otherwise returns `true`. -If "makie" is found in `so_filename`, also checks against the source code in +If `so_filename` is `nothing`, use the name of the system image of the current julia +session for `so_filename`. + +If `so_filename` is `"makie_postproc.so"`, also checks against the source code in `makie_post_processing/makie_post_processing/src/`. -If "plots" is found in `so_filename`, also checks against the source code in +If `so_filename` is `"plots_postproc.so"`, also checks against the source code in `plots_post_processing/plots_post_processing/src/`. """ -function check_so_newer_than_code(so_filename="moment_kinetics.so") - is_makie = occursin("makie", so_filename) - is_plots = occursin("plots", so_filename) +function check_so_newer_than_code(so_filename=nothing) + if so_filename === nothing + # Get filename of system image currently being used. + # https://discourse.julialang.org/t/get-path-of-system-image-from-within-julia/108257/2 + so_filename = unsafe_string(Base.JLOptions().image_file) + end + + if basename(so_filename) ∉ ("moment_kinetics.so", "makie_postproc.so", "plots_postproc.so") + # Not using a custom system image that includes moment_kinetics, so no need to + # check. + return nothing + end + + is_makie = (basename(so_filename) == "makie_postproc.so") + is_plots = (basename(so_filename) == "plots_postproc.so") if !isfile(so_filename) error("Trying to check age of $so_filename, but $(realpath(so_filename)) does " @@ -57,9 +72,11 @@ function check_so_newer_than_code(so_filename="moment_kinetics.so") if !so_is_newer error_message = - "WARNING: source code files have been modified more recently than\n" * - "'$so_filename'. It is likely that you need to re-compile your system image\n" * - "(so that the code changes take effect) by re-running " + "\n************************ WARNING ************************\n" * + " source code files have been modified more recently than\n" * + "'$so_filename'.\n" * + "It is likely that you need to re-compile your system image\n" * + "(so that the code changes take effect) by re-running\n" if is_makie error_message *= "'precompile-makie-post-processing-submit.sh'" * "\n(or 'precompile_submit.sh')." @@ -69,6 +86,7 @@ function check_so_newer_than_code(so_filename="moment_kinetics.so") else error_message *= "'precompile-submit.sh'." end + error_message *= "\n" println(error_message) end diff --git a/moment_kinetics/src/moment_kinetics.jl b/moment_kinetics/src/moment_kinetics.jl index 7c580d441..d3c545b34 100644 --- a/moment_kinetics/src/moment_kinetics.jl +++ b/moment_kinetics/src/moment_kinetics.jl @@ -97,6 +97,16 @@ main function that contains all of the content of the program """ function run_moment_kinetics(to::Union{TimerOutput,Nothing}, input_dict=Dict(); restart=false, restart_time_index=-1) + + if global_rank[] == 0 + # Check that, if we are using a custom compiled system image that includes + # moment_kinetics, the system image is newer than the source code files (if there + # are changes made to the source code since the system image was compiled, they + # will not affect the current run). Prints a warning if any code files are newer + # than the system image. + check_so_newer_than_code() + end + mk_state = nothing try # set up all the structs, etc. needed for a run diff --git a/plots_post_processing/plots_post_processing/src/plots_post_processing.jl b/plots_post_processing/plots_post_processing/src/plots_post_processing.jl index 7303979d6..149355eee 100644 --- a/plots_post_processing/plots_post_processing/src/plots_post_processing.jl +++ b/plots_post_processing/plots_post_processing/src/plots_post_processing.jl @@ -32,6 +32,7 @@ using LaTeXStrings using Measures using MPI # modules +using moment_kinetics: check_so_newer_than_code using moment_kinetics.communication using moment_kinetics.quadrature: composite_simpson_weights using moment_kinetics.array_allocation: allocate_float @@ -313,6 +314,14 @@ does not have a `_` suffix). Note that `run_index` is only used when a direct (rather than the prefix of a specific output file) is passed to `prefix...` """ function analyze_and_plot_data(prefix...; run_index=nothing) + + # Check that, if we are using a custom compiled system image that includes + # moment_kinetics, the system image is newer than the source code files (if there are + # changes made to the source code since the system image was compiled, they will not + # affect the current run). Prints a warning if any code files are newer than the + # system image. + check_so_newer_than_code() + if length(prefix) == 0 println("No runs to plot!") return nothing From 88fca48fbbded02896730e63e4ff3a6ddf3ab878 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 3 Jan 2024 12:42:54 +0000 Subject: [PATCH 69/80] Fix title on moment_constraints API docs page --- docs/src/zz_moment_constraints.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/zz_moment_constraints.md b/docs/src/zz_moment_constraints.md index 6d15a124e..f5c788dd3 100644 --- a/docs/src/zz_moment_constraints.md +++ b/docs/src/zz_moment_constraints.md @@ -1,5 +1,5 @@ -`moment_kinetics` -================= +`moment_constraints` +==================== ```@autodocs Modules = [moment_kinetics.moment_constraints] From ff53e685e553eb960721b99dc0fc0bd3fb071198 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 4 Jan 2024 10:21:24 +0000 Subject: [PATCH 70/80] Fix for using default path for .julia directory When $JULIA_DIRECTORY is empty, need to no set $JULIA_DEPOT_PATH. --- machines/machine_setup.sh | 4 +++- machines/shared/machine_setup.jl | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/machines/machine_setup.sh b/machines/machine_setup.sh index 3fb6c1d54..2f85133b0 100755 --- a/machines/machine_setup.sh +++ b/machines/machine_setup.sh @@ -245,7 +245,9 @@ echo # export JULIA_DEPOT_PATH instead of just passing as a prefix to the julia # command, because passing as a prefix does not work (sometimes??) within a # bash script (even though as far as JTO knows it should work). -export JULIA_DEPOT_PATH=$JULIA_DIRECTORY +if [ ! -z "$JULIA_DIRECTORY" ]; then + export JULIA_DEPOT_PATH=$JULIA_DIRECTORY +fi $JULIA --project machines/shared/machine_setup.jl "$MACHINE" if [ -f julia.env ]; then diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index 5e9dfe075..75104a56a 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -185,7 +185,12 @@ function machine_setup_moment_kinetics(machine::String; no_force_exit::Bool=fals mk_preferences["batch_system"] = batch_system # Get some settings - julia_directory = mk_preferences["julia_directory"] = ENV["JULIA_DEPOT_PATH"] + if haskey(ENV, "JULIA_DEPOT_PATH") + julia_directory = ENV["JULIA_DEPOT_PATH"] + else + julia_directory = "" + end + mk_preferences["julia_directory"] = julia_directory if batch_system get_setting("default_run_time", "Enter the default value for the time limit for simulation jobs", From 391bfd86990104739fb4bf8a0b3bccebaa740635 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 4 Jan 2024 10:22:20 +0000 Subject: [PATCH 71/80] Fix typo getting hdf5_dir --- machines/shared/add_dependencies_to_project.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/shared/add_dependencies_to_project.jl b/machines/shared/add_dependencies_to_project.jl index 20cfcc1b7..29d479418 100644 --- a/machines/shared/add_dependencies_to_project.jl +++ b/machines/shared/add_dependencies_to_project.jl @@ -122,7 +122,7 @@ elseif machine_settings["hdf5_library_setting"] == "prompt" hdf5_lib = joinpath(local_hdf5_install_dir, "libhdf5.so") hdf5_lib_hl = joinpath(local_hdf5_install_dir, "libhdf5_hl.so") elseif !prompt_for_hdf5 - hdf5_dir = mk_preferences("hdf5_dir") + hdf5_dir = mk_preferences["hdf5_dir"] if hdf5_dir != "default" hdf5_lib = joinpath(hdf5_dir, "libhdf5.so") hdf5_lib_hl = joinpath(hdf5_dir, "libhdf5_hl.so") From 1a03998e9a526d0ec26c4ba95566f76fcd6e486c Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 4 Jan 2024 10:23:40 +0000 Subject: [PATCH 72/80] Fix prompt for BUILDHDF5 --- machines/generic-pc/compile_dependencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machines/generic-pc/compile_dependencies.sh b/machines/generic-pc/compile_dependencies.sh index 3add3b031..852a1d451 100755 --- a/machines/generic-pc/compile_dependencies.sh +++ b/machines/generic-pc/compile_dependencies.sh @@ -41,9 +41,9 @@ else read -p "> " input done if [[ -z $input || $input == "n" ]]; then - BUILDHDF5="y" - else BUILDHDF5="n" + else + BUILDHDF5="y" fi fi From de2d60c7a8da6001475a65a3ffa5d92c908f1368 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 4 Jan 2024 12:11:42 +0000 Subject: [PATCH 73/80] Update test commands in docs for new package structure --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 119667199..40f068ece 100644 --- a/README.md +++ b/README.md @@ -338,28 +338,28 @@ $ julia -p 8 -O3 --project run_parameter_scan.jl path/to/scan/input.toml There is a test suite in the `test/` subdirectory. It can be run in a few ways: * Execute some or all of the tests as a script. For example in the terminal run ``` - $ julia -O3 --project test/runtests.jl + $ julia -O3 --project moment_kinetcs/test/runtests.jl ``` or in the REPL run ``` - julia> include("test/runtests.jl") + julia> include("moment_kinetcs/test/runtests.jl") ``` Individual test files can also be used instead of `runtests.jl`, which runs all the tests. * You can also run the tests using `Pkg`. Either using `pkg>` mode ``` $ julia -O3 --project julia> - (moment_kinetics) pkg> test + (moment_kinetics) pkg> test moment_kinetics ``` using `Pkg` in the REPL ``` $ julia -O3 --project julia> import Pkg - julia> Pkg.test() + julia> Pkg.test("moment_kinetics") ``` or run on the command line ``` - julia -O3 --project -e "import Pkg; Pkg.test()` + julia -O3 --project -e "import Pkg; Pkg.test("moment_kinetics")` ``` The downside of this method is that it will cause `NCDatasets` to be installed if you did not install it already, which might sometimes cause @@ -376,11 +376,11 @@ so, it skips many cases. To run more comprehensive tests, you can activate the before running the tests. * Running from the terminal, pass as a command line argument, e.g. ``` - $ julia -O3 --project --long test/runtests.jl + $ julia -O3 --project --long moment_kinetics/test/runtests.jl ``` * Using `test_args` argument ``` - julia> Pkg.test(; test_args=["--long"]) + julia> Pkg.test("moment_kinetics"; test_args=["--long"]) ``` Note the semicolon is necessary. From 523dca94fed61d2c34c44fd89fab2b101033f677 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Fri, 5 Jan 2024 16:22:34 +0000 Subject: [PATCH 74/80] Fix debug tests so that they can run from top-level project Previously the imports of Glob and Primes packages required that `moment_kinetics` was the active project, not just added as a dependency. --- moment_kinetics/debug_test/runtest_template.jl | 4 ++-- moment_kinetics/src/moment_kinetics.jl | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/moment_kinetics/debug_test/runtest_template.jl b/moment_kinetics/debug_test/runtest_template.jl index 7d7928e3a..327280382 100644 --- a/moment_kinetics/debug_test/runtest_template.jl +++ b/moment_kinetics/debug_test/runtest_template.jl @@ -2,8 +2,8 @@ using moment_kinetics: setup_moment_kinetics, cleanup_moment_kinetics! using moment_kinetics.time_advance: time_advance! using moment_kinetics.communication using moment_kinetics.looping: dimension_combinations -using Glob -using Primes +using moment_kinetics.Glob +using moment_kinetics.Primes """ Run a test for a single set of parameters diff --git a/moment_kinetics/src/moment_kinetics.jl b/moment_kinetics/src/moment_kinetics.jl index d3c545b34..80c73ace1 100644 --- a/moment_kinetics/src/moment_kinetics.jl +++ b/moment_kinetics/src/moment_kinetics.jl @@ -69,6 +69,7 @@ include("time_advance.jl") using TimerOutputs using Dates using Glob +using Primes using .file_io: setup_file_io, finish_file_io using .file_io: write_data_to_ascii From 6fb0a09d00d4fb116f5e46dfc44ec7bd8c2336d5 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 7 Jan 2024 11:50:42 +0000 Subject: [PATCH 75/80] Fix vperp inputs in `util/precompile_run.jl` --- util/precompile_run.jl | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/util/precompile_run.jl b/util/precompile_run.jl index bd77578b9..df6c072a6 100644 --- a/util/precompile_run.jl +++ b/util/precompile_run.jl @@ -20,19 +20,29 @@ base_input = Dict("nstep"=>1, "z_nelement" => 1, "z_bc" => "periodic", "z_discretization" => "finite_difference", - "verp_ngrid" => 5, - "verp_nelement" => 1, - "verp_bc" => "periodic", - "verp_L" => 4.0, - "verp_discretization" => "finite_difference", + "vperp_ngrid" => 5, + "vperp_nelement" => 1, + "vperp_bc" => "zero", + "vperp_L" => 4.0, + "vperp_discretization" => "finite_difference", "vpa_ngrid" => 7, "vpa_nelement" => 1, - "vpa_bc" => "periodic", + "vpa_bc" => "zero", "vpa_L" => 4.0, "vpa_discretization" => "finite_difference", + "vzeta_ngrid" => 5, + "vzeta_nelement" => 1, + "vzeta_bc" => "zero", + "vzeta_L" => 4.0, + "vzeta_discretization" => "finite_difference", + "vr_ngrid" => 5, + "vr_nelement" => 1, + "vr_bc" => "zero", + "vr_L" => 4.0, + "vr_discretization" => "finite_difference", "vz_ngrid" => 7, "vz_nelement" => 1, - "vz_bc" => "periodic", + "vz_bc" => "zero", "vz_L" => 4.0, "vz_discretization" => "finite_difference") cheb_input = merge(base_input, Dict("r_discretization" => "chebyshev_pseudospectral", @@ -47,7 +57,8 @@ for input ∈ [base_input, cheb_input, wall_bc_input, wall_bc_cheb_input] push!(inputs_list, input) x = merge(input, Dict("evolve_moments_density" => true, "ionization_frequency" => 0.0, "r_ngrid" => 1, "r_nelement" => 1, "vperp_ngrid" => 1, - "vperp_nelement" => 1)) + "vperp_nelement" => 1, "vzeta_ngrid" => 1, + "vzeta_nelement" => 1, "vr_ngrid" => 1, "vr_nelement" => 1)) push!(inputs_list, x) x = merge(x, Dict("evolve_moments_parallel_flow" => true)) push!(inputs_list, x) From 70665a3fd0b5d80ce22a8018b46b8802a84bfa11 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 7 Jan 2024 11:51:12 +0000 Subject: [PATCH 76/80] Increase nelement to 3 in `util/precompile_run.jl` There may be some routines that follow different code paths for the first or last element, so try to make sure all code is used. --- util/precompile_run.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/util/precompile_run.jl b/util/precompile_run.jl index df6c072a6..9482f961a 100644 --- a/util/precompile_run.jl +++ b/util/precompile_run.jl @@ -13,35 +13,35 @@ base_input = Dict("nstep"=>1, "base_directory" => test_output_directory, "dt" => 0.0, "r_ngrid" => 5, - "r_nelement" => 1, + "r_nelement" => 3, "r_bc" => "periodic", "r_discretization" => "finite_difference", "z_ngrid" => 5, - "z_nelement" => 1, + "z_nelement" => 3, "z_bc" => "periodic", "z_discretization" => "finite_difference", "vperp_ngrid" => 5, - "vperp_nelement" => 1, + "vperp_nelement" => 3, "vperp_bc" => "zero", "vperp_L" => 4.0, "vperp_discretization" => "finite_difference", "vpa_ngrid" => 7, - "vpa_nelement" => 1, + "vpa_nelement" => 3, "vpa_bc" => "zero", "vpa_L" => 4.0, "vpa_discretization" => "finite_difference", "vzeta_ngrid" => 5, - "vzeta_nelement" => 1, + "vzeta_nelement" => 3, "vzeta_bc" => "zero", "vzeta_L" => 4.0, "vzeta_discretization" => "finite_difference", "vr_ngrid" => 5, - "vr_nelement" => 1, + "vr_nelement" => 3, "vr_bc" => "zero", "vr_L" => 4.0, "vr_discretization" => "finite_difference", "vz_ngrid" => 7, - "vz_nelement" => 1, + "vz_nelement" => 3, "vz_bc" => "zero", "vz_L" => 4.0, "vz_discretization" => "finite_difference") From 1f0f3d01259166eb52ae3102fd06f58c250596ce Mon Sep 17 00:00:00 2001 From: John Omotani Date: Sun, 7 Jan 2024 11:51:54 +0000 Subject: [PATCH 77/80] Run with Fokker-Planck collision operator in `util/precompile_run.jl` Means collision operator code gets compiled and added to system image `moment_kinetics.so`. --- util/precompile_run.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/util/precompile_run.jl b/util/precompile_run.jl index 9482f961a..1e0b040b5 100644 --- a/util/precompile_run.jl +++ b/util/precompile_run.jl @@ -66,6 +66,13 @@ for input ∈ [base_input, cheb_input, wall_bc_input, wall_bc_cheb_input] push!(inputs_list, x) end +collisions_input = merge(wall_bc_cheb_input, Dict("n_neutral_species" => 0, + "nuii" => 1.0, + "vperp_discretization" => "gausslegendre_pseudospectral", + "vpa_discretization" => "gausslegendre_pseudospectral", + )) +push!(inputs_list, collisions_input) + for input in inputs_list run_moment_kinetics(input) end From 054431bbdfeb075b8c47bda1047efedbe59732b4 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 1 Feb 2024 14:21:44 +0000 Subject: [PATCH 78/80] Don't make bin/julia a symlink when enabling plots_post_processing Needs to be a bash script so we can activate the Python venv within it when enabling plots_post_processing. --- machines/shared/machine_setup.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/machines/shared/machine_setup.jl b/machines/shared/machine_setup.jl index 75104a56a..c93696f78 100644 --- a/machines/shared/machine_setup.jl +++ b/machines/shared/machine_setup.jl @@ -301,7 +301,8 @@ function machine_setup_moment_kinetics(machine::String; no_force_exit::Bool=fals bindir = joinpath(repo_dir, "bin") mkpath(bindir) julia_executable_name = joinpath(bindir, "julia") - if batch_system || julia_directory == "" + println("check use_plots=", mk_preferences["use_plots"], " ", mk_preferences["use_plots"] == "n") + if batch_system || (julia_directory == "" && mk_preferences["use_plots"] == "n") # Make a local link to the Julia binary so scripts in the repo can find it println("\n** Making a symlink to the julia executable at bin/julia\n") islink(julia_executable_name) && rm(julia_executable_name) @@ -311,7 +312,9 @@ function machine_setup_moment_kinetics(machine::String; no_force_exit::Bool=fals # needing the julia.env setup open(julia_executable_name, "w") do io println(io, "#!/usr/bin/env bash") - println(io, "export JULIA_DEPOT_PATH=$julia_directory") + if julia_directory != "" + println(io, "export JULIA_DEPOT_PATH=$julia_directory") + end julia_path = joinpath(Sys.BINDIR, "julia") println(io, "$julia_path \"\$@\"") end From c0bf27f763f1143412fae86318da7f2b5dcf1aba Mon Sep 17 00:00:00 2001 From: John Omotani Date: Thu, 1 Feb 2024 14:57:31 +0000 Subject: [PATCH 79/80] Restore split into separate get_geometry() and get_composition() --- .../makie_post_processing/src/makie_post_processing.jl | 2 +- .../makie_post_processing/src/shared_utils.jl | 10 ++++++++-- .../plots_post_processing/src/plots_post_processing.jl | 9 +++++---- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/makie_post_processing/makie_post_processing/src/makie_post_processing.jl b/makie_post_processing/makie_post_processing/src/makie_post_processing.jl index 2e2b26369..cbfa90211 100644 --- a/makie_post_processing/makie_post_processing/src/makie_post_processing.jl +++ b/makie_post_processing/makie_post_processing/src/makie_post_processing.jl @@ -44,7 +44,7 @@ using moment_kinetics.load_data: close_run_info, get_run_info_no_setup, get_vari neutral_dfn_variables, all_dfn_variables, ion_variables, neutral_variables, all_variables using moment_kinetics.initial_conditions: vpagrid_to_dzdt -using .shared_utils: calculate_and_write_frequencies, get_geometry_and_composition +using .shared_utils: calculate_and_write_frequencies using moment_kinetics.type_definitions: mk_float, mk_int using moment_kinetics.velocity_moments: integrate_over_vspace, integrate_over_neutral_vspace diff --git a/makie_post_processing/makie_post_processing/src/shared_utils.jl b/makie_post_processing/makie_post_processing/src/shared_utils.jl index 4e4a4c17e..076e8010e 100644 --- a/makie_post_processing/makie_post_processing/src/shared_utils.jl +++ b/makie_post_processing/makie_post_processing/src/shared_utils.jl @@ -63,7 +63,7 @@ end """ """ -function get_geometry_and_composition(scan_input, z, r) +function get_geometry(scan_input, z, r) # set geometry reference_params = setup_reference_parameters(scan_input) # set geometry_input @@ -76,6 +76,12 @@ function get_geometry_and_composition(scan_input, z, r) geo_in = geometry_input(rhostar,option,pitch,DeltaB) geometry = init_magnetic_geometry(geo_in,z,r) + return geometry +end + +""" +""" +function get_composition(scan_input) # set composition input # MRH need to get this in way that does not duplicate code # MRH from moment_kinetics_input.jl @@ -116,7 +122,7 @@ function get_geometry_and_composition(scan_input, z, r) composition = species_composition(n_species, n_ion_species, n_neutral_species, electron_physics, use_test_neutral_wall_pdf, T_e, T_wall, phi_wall, Er_constant, mn_over_mi, me_over_mi, recycling_fraction, allocate_float(n_species)) - return geometry, composition + return composition end diff --git a/plots_post_processing/plots_post_processing/src/plots_post_processing.jl b/plots_post_processing/plots_post_processing/src/plots_post_processing.jl index f48084241..11bcaac08 100644 --- a/plots_post_processing/plots_post_processing/src/plots_post_processing.jl +++ b/plots_post_processing/plots_post_processing/src/plots_post_processing.jl @@ -66,7 +66,7 @@ using moment_kinetics.input_structs: electron_physics_type, boltzmann_electron_r using moment_kinetics.reference_parameters using moment_kinetics.geo: init_magnetic_geometry using .post_processing_input: pp -using .shared_utils: calculate_and_write_frequencies, get_geometry_and_composition +using .shared_utils: calculate_and_write_frequencies, get_geometry, get_composition using TOML import Base: get @@ -640,9 +640,10 @@ function analyze_and_plot_data(prefix...; run_index=nothing) end end - geometry, composition = - get_tuple_of_return_values(get_geometry_and_composition, scan_input, - z, r) + geometry = + get_tuple_of_return_values(get_geometry, scan_input, z, r) + composition = + get_tuple_of_return_values(get_composition, scan_input) # initialise the post-processing input options nwrite_movie, itime_min, itime_max, nwrite_movie_pdfs, itime_min_pdfs, itime_max_pdfs, From 6e9e2e39cad371c738f8990e169694a24c9243ce Mon Sep 17 00:00:00 2001 From: John Omotani Date: Fri, 2 Feb 2024 12:43:38 +0000 Subject: [PATCH 80/80] Fix export of get_geometry() and get_composition() --- makie_post_processing/makie_post_processing/src/shared_utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makie_post_processing/makie_post_processing/src/shared_utils.jl b/makie_post_processing/makie_post_processing/src/shared_utils.jl index 076e8010e..bdaf99e53 100644 --- a/makie_post_processing/makie_post_processing/src/shared_utils.jl +++ b/makie_post_processing/makie_post_processing/src/shared_utils.jl @@ -1,6 +1,6 @@ module shared_utils -export calculate_and_write_frequencies, get_geometry_and_composition +export calculate_and_write_frequencies, get_geometry, get_composition using moment_kinetics.analysis: fit_delta_phi_mode using moment_kinetics.array_allocation: allocate_float