diff --git a/hail/python/benchmark/hail/benchmark_benchmark_analysis.py b/hail/python/benchmark/hail/benchmark_benchmark_analysis.py index 2b1c147f9c2f..d0a15e7dae82 100644 --- a/hail/python/benchmark/hail/benchmark_benchmark_analysis.py +++ b/hail/python/benchmark/hail/benchmark_benchmark_analysis.py @@ -7,7 +7,7 @@ from benchmark.tools.statistics import analyze_benchmarks -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=5, iterations=10, burn_in_iterations=10) def benchmark_analyze_benchmarks(local_tmpdir, onethreetwo, onethreethree): inputs = (onethreetwo, onethreethree) inputs = ((v, Path(tempfile.mktemp(dir=local_tmpdir))) for v in inputs) diff --git a/hail/python/benchmark/hail/benchmark_combiner.py b/hail/python/benchmark/hail/benchmark_combiner.py index 1d6848d08e38..cc647c061f48 100644 --- a/hail/python/benchmark/hail/benchmark_combiner.py +++ b/hail/python/benchmark/hail/benchmark_combiner.py @@ -25,14 +25,14 @@ def benchmark_compile_2k_merge(empty_gvcf, tmp_path): hl.vds.write_variant_datasets(combined, str(tmp_path / 'combiner-multi-write'), overwrite=True) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=10, iterations=10, burn_in_iterations=10) @pytest.mark.xfail(raises=TimeoutError, reason=XFail.Timeout) def benchmark_python_only_10k_transform(empty_gvcf): for vcf in [import_vcf(empty_gvcf)] * 10_000: transform_gvcf(vcf, []) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=10, iterations=5, burn_in_iterations=20) def benchmark_python_only_10k_combine(empty_gvcf): vcf = import_vcf(empty_gvcf) mt = transform_gvcf(vcf, []) @@ -40,7 +40,7 @@ def benchmark_python_only_10k_combine(empty_gvcf): combine_variant_datasets(mts) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=10, burn_in_iterations=10) def benchmark_import_and_transform_gvcf(single_gvcf): mt = import_vcf(single_gvcf) vds = transform_gvcf(mt, []) @@ -48,7 +48,7 @@ def benchmark_import_and_transform_gvcf(single_gvcf): vds.variant_data._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=15, burn_in_iterations=8) def benchmark_import_gvcf_force_count(single_gvcf): mt = import_vcf(single_gvcf) mt._force_count_rows() @@ -63,7 +63,7 @@ def tmp_and_output_paths(tmp_path): return (tmp, output) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=10, iterations=5, burn_in_iterations=10) @pytest.mark.xfail(raises=TimeoutError, reason=XFail.Timeout) def benchmark_vds_combiner_chr22(chr22_gvcfs, tmp_and_output_paths): parts = hl.eval([hl.parse_locus_interval('chr22:start-end', reference_genome='GRCh38')]) diff --git a/hail/python/benchmark/hail/benchmark_linalg.py b/hail/python/benchmark/hail/benchmark_linalg.py index ea66253fcbee..473b6a346e6c 100644 --- a/hail/python/benchmark/hail/benchmark_linalg.py +++ b/hail/python/benchmark/hail/benchmark_linalg.py @@ -13,32 +13,32 @@ def benchmark_block_matrix_nested_multiply(tmp_path): bm.write(str(tmp_path / 'result.mt'), overwrite=True) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=5, burn_in_iterations=5) def benchmark_make_ndarray(): ht = hl.utils.range_table(200_000) ht = ht.annotate(x=hl.nd.array(hl.range(ht.idx))) ht._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=20, burn_in_iterations=10) def benchmark_ndarray_addition(): arr = hl.nd.ones((1024, 1024)) hl.eval(arr + arr) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=20, iterations=5, burn_in_iterations=10) def benchmark_ndarray_matmul_int64(): arr = hl.nd.arange(1024 * 1024).map(hl.int64).reshape((1024, 1024)) hl.eval(arr @ arr) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=15, burn_in_iterations=6) def benchmark_ndarray_matmul_float64(): arr = hl.nd.arange(1024 * 1024).map(hl.float64).reshape((1024, 1024)) hl.eval(arr @ arr) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=5, burn_in_iterations=10) @pytest.mark.xfail(raises=TimeoutError, reason=XFail.Timeout) def benchmark_blockmatrix_write_from_entry_expr_range_mt(tmp_path): mt = hl.utils.range_matrix_table(40_000, 40_000, n_partitions=4) @@ -56,7 +56,7 @@ def benchmark_blockmatrix_write_from_entry_expr_range_mt_standardize(tmp_path): ) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=5, iterations=5, burn_in_iterations=10) def benchmark_sum_table_of_ndarrays(): ht = hl.utils.range_table(400).annotate(nd=hl.nd.ones((4096, 4096))) ht.aggregate(hl.agg.ndarray_sum(ht.nd)) diff --git a/hail/python/benchmark/hail/benchmark_matrix_table.py b/hail/python/benchmark/hail/benchmark_matrix_table.py index 43d2dac0dffd..b00b16e7114f 100644 --- a/hail/python/benchmark/hail/benchmark_matrix_table.py +++ b/hail/python/benchmark/hail/benchmark_matrix_table.py @@ -4,110 +4,110 @@ from benchmark.hail.utils import XFail -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=25, iterations=15, burn_in_iterations=8) def benchmark_matrix_table_decode_and_count(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=20, burn_in_iterations=5) def benchmark_matrix_table_decode_and_count_just_gt(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)).select_entries('GT') mt._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=25, iterations=10, burn_in_iterations=20) def benchmark_matrix_table_array_arithmetic(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt = mt.filter_rows(mt.alleles.length() == 2) mt.select_entries(dosage=hl.pl_dosage(mt.PL)).select_rows()._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=25, iterations=5, burn_in_iterations=10) def benchmark_matrix_table_entries_table(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt.entries()._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=10, burn_in_iterations=10) def benchmark_matrix_table_entries_table_no_key(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)).key_rows_by().key_cols_by() mt.entries()._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=10, burn_in_iterations=30) def benchmark_matrix_table_rows_force_count(profile25_mt): ht = hl.read_matrix_table(str(profile25_mt)).rows().key_by() ht._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=15, iterations=10, burn_in_iterations=15) def benchmark_matrix_table_show(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt.show(100) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=10, burn_in_iterations=15) def benchmark_matrix_table_rows_show(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt.rows().show(100) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=15, iterations=15, burn_in_iterations=16) def benchmark_matrix_table_cols_show(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt.cols().show(100) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=15, burn_in_iterations=10) def benchmark_matrix_table_take_entry(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt.GT.take(100) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=15, burn_in_iterations=15) def benchmark_matrix_table_entries_show(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt.entries().show() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=10, iterations=20, burn_in_iterations=10) def benchmark_matrix_table_take_row(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt.info.AF.take(100) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=15, iterations=20, burn_in_iterations=10) def benchmark_matrix_table_take_col(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt.s.take(100) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=10, burn_in_iterations=8) def benchmark_write_range_matrix_table_p100(tmp_path): mt = hl.utils.range_matrix_table(n_rows=1_000_000, n_cols=10, n_partitions=100) mt = mt.annotate_entries(x=mt.col_idx + mt.row_idx) mt.write(str(tmp_path / 'tmp.mt')) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=15, burn_in_iterations=15) def benchmark_write_profile_mt(profile25_mt, tmp_path): hl.read_matrix_table(str(profile25_mt)).write(str(tmp_path / 'tmp.mt')) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=20, burn_in_iterations=9) def benchmark_matrix_table_rows_is_transition(profile25_mt): ht = hl.read_matrix_table(str(profile25_mt)).rows().key_by() ht.select(is_snp=hl.is_snp(ht.alleles[0], ht.alleles[1]))._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=15, iterations=20, burn_in_iterations=6) def benchmark_matrix_table_filter_entries(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt.filter_entries((mt.GQ > 8) & (mt.DP > 2))._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=10, burn_in_iterations=3) def benchmark_matrix_table_filter_entries_unfilter(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt.filter_entries((mt.GQ > 8) & (mt.DP > 2)).unfilter_entries()._force_count_rows() @@ -164,27 +164,27 @@ def many_aggs(mt): return {f'x{i}': expr for i, expr in enumerate(aggs)} -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=5, burn_in_iterations=4) def benchmark_matrix_table_many_aggs_row_wise(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt = mt.annotate_rows(**many_aggs(mt)) mt.rows()._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=5, burn_in_iterations=10) def benchmark_matrix_table_many_aggs_col_wise(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt = mt.annotate_cols(**many_aggs(mt)) mt.cols()._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=15, burn_in_iterations=8) def benchmark_matrix_table_aggregate_entries(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt.aggregate_entries(hl.agg.stats(mt.GQ)) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=10, burn_in_iterations=8) def benchmark_matrix_table_call_stats_star_star(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt.annotate_rows(**hl.agg.call_stats(mt.GT, mt.alleles))._force_count_rows() @@ -242,24 +242,24 @@ def benchmark_gnomad_coverage_stats_optimized(gnomad_dp_sim): mt.rows()._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=15, iterations=20, burn_in_iterations=10) def benchmark_per_row_stats_star_star(gnomad_dp_sim): mt = hl.read_matrix_table(str(gnomad_dp_sim)) mt.annotate_rows(**hl.agg.stats(mt.x))._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=10, burn_in_iterations=10) def benchmark_read_decode_gnomad_coverage(gnomad_dp_sim): hl.read_matrix_table(str(gnomad_dp_sim))._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=10, iterations=5, burn_in_iterations=10) def benchmark_import_bgen_force_count_just_gp(sim_ukb_bgen, sim_ukb_sample): mt = hl.import_bgen(str(sim_ukb_bgen), sample_file=str(sim_ukb_sample), entry_fields=['GP'], n_partitions=8) mt._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=10, iterations=5, burn_in_iterations=20) def benchmark_import_bgen_force_count_all(sim_ukb_bgen, sim_ukb_sample): mt = hl.import_bgen( str(sim_ukb_bgen), sample_file=str(sim_ukb_sample), entry_fields=['GT', 'GP', 'dosage'], n_partitions=8 @@ -267,7 +267,7 @@ def benchmark_import_bgen_force_count_all(sim_ukb_bgen, sim_ukb_sample): mt._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=10, iterations=5, burn_in_iterations=12) @pytest.mark.xfail(raises=TimeoutError, reason=XFail.Timeout) def benchmark_import_bgen_info_score(sim_ukb_bgen, sim_ukb_sample): mt = hl.import_bgen(str(sim_ukb_bgen), sample_file=str(sim_ukb_sample), entry_fields=['GP'], n_partitions=8) @@ -275,27 +275,27 @@ def benchmark_import_bgen_info_score(sim_ukb_bgen, sim_ukb_sample): mt.rows().select('info_score')._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=10, iterations=5, burn_in_iterations=18) def benchmark_import_bgen_filter_count(sim_ukb_bgen, sim_ukb_sample): mt = hl.import_bgen(str(sim_ukb_bgen), sample_file=str(sim_ukb_sample), entry_fields=['GT', 'GP'], n_partitions=8) mt = mt.filter_rows(mt.alleles == ['A', 'T']) mt._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=15, iterations=20, burn_in_iterations=3) def benchmark_export_range_matrix_table_entry_field_p100(tmp_path): mt = hl.utils.range_matrix_table(n_rows=1_000_000, n_cols=10, n_partitions=100) mt = mt.annotate_entries(x=mt.col_idx + mt.row_idx) mt.x.export(str(tmp_path / 'result.txt')) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=10, burn_in_iterations=8) def benchmark_export_range_matrix_table_row_p100(tmp_path): mt = hl.utils.range_matrix_table(n_rows=1_000_000, n_cols=10, n_partitions=100) mt.row.export(str(tmp_path / 'result.txt')) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=15, iterations=25, burn_in_iterations=15) def benchmark_export_range_matrix_table_col_p100(tmp_path): mt = hl.utils.range_matrix_table(n_rows=1_000_000, n_cols=10, n_partitions=100) mt.col.export(str(tmp_path / 'result.txt')) @@ -309,7 +309,7 @@ def benchmark_large_range_matrix_table_sum(): mt.annotate_cols(foo=hl.agg.sum(mt.x))._force_count_cols() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=5, burn_in_iterations=7) def benchmark_kyle_sex_specific_qc(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt = mt.annotate_cols(sex=hl.if_else(hl.rand_bool(0.5), 'Male', 'Female')) @@ -350,14 +350,14 @@ def benchmark_kyle_sex_specific_qc(profile25_mt): mt.rows()._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=25, iterations=10, burn_in_iterations=5) def benchmark_matrix_table_scan_count_rows_2(): mt = hl.utils.range_matrix_table(n_rows=200_000_000, n_cols=10, n_partitions=16) mt = mt.annotate_rows(x=hl.scan.count()) mt._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.3, instances=20, iterations=10, burn_in_iterations=20) def benchmark_matrix_table_scan_count_cols_2(): mt = hl.utils.range_matrix_table(n_cols=10_000_000, n_rows=10) mt = mt.annotate_cols(x=hl.scan.count()) @@ -372,14 +372,14 @@ def benchmark_matrix_multi_write_nothing(tmp_path): hl.experimental.write_matrix_tables(mts, str(tmp_path / 'multi-write')) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=25, iterations=10, burn_in_iterations=5) def benchmark_mt_localize_and_collect(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) ht = mt.localize_entries("ent") ht.head(150).collect() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=15, burn_in_iterations=5) def benchmark_mt_group_by_memory_usage(random_doubles_mt): mt = hl.read_matrix_table(str(random_doubles_mt)) mt = mt.group_rows_by(new_idx=mt.row_idx % 3).aggregate(x=hl.agg.mean(mt.x)) diff --git a/hail/python/benchmark/hail/benchmark_methods.py b/hail/python/benchmark/hail/benchmark_methods.py index 43a79e275188..4a251f146612 100644 --- a/hail/python/benchmark/hail/benchmark_methods.py +++ b/hail/python/benchmark/hail/benchmark_methods.py @@ -4,43 +4,43 @@ from benchmark.hail.utils import XFail -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=10, burn_in_iterations=5) def benchmark_import_vcf_write(profile25_vcf, tmp_path): mt = hl.import_vcf(str(profile25_vcf)) out = str(tmp_path / 'out.mt') mt.write(out) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=5, burn_in_iterations=1) def benchmark_import_vcf_count_rows(profile25_vcf): mt = hl.import_vcf(str(profile25_vcf)) mt.count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=15, iterations=5, burn_in_iterations=15) def benchmark_export_vcf(profile25_mt, tmp_path): mt = hl.read_matrix_table(str(profile25_mt)) out = str(tmp_path / 'out.vcf.bgz') hl.export_vcf(mt, out) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=5, burn_in_iterations=10) def benchmark_sample_qc(profile25_mt): hl.sample_qc(hl.read_matrix_table(str(profile25_mt))).cols()._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=25, iterations=5, burn_in_iterations=8) def benchmark_variant_qc(profile25_mt): hl.variant_qc(hl.read_matrix_table(str(profile25_mt))).rows()._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=10, iterations=5, burn_in_iterations=18) def benchmark_variant_and_sample_qc(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) hl.sample_qc(hl.variant_qc(mt))._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=15, iterations=15, burn_in_iterations=13) def benchmark_variant_and_sample_qc_nested_with_filters_2(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt = hl.variant_qc(mt) @@ -54,7 +54,7 @@ def benchmark_variant_and_sample_qc_nested_with_filters_2(profile25_mt): mt.count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=10, iterations=5, burn_in_iterations=20) def benchmark_variant_and_sample_qc_nested_with_filters_4(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt = hl.variant_qc(mt) @@ -76,7 +76,7 @@ def benchmark_variant_and_sample_qc_nested_with_filters_4(profile25_mt): mt.count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=15, iterations=10, burn_in_iterations=20) def benchmark_variant_and_sample_qc_nested_with_filters_4_counts(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt = hl.variant_qc(mt) @@ -108,7 +108,7 @@ def benchmark_hwe_normalized_pca(profile25_mt): hl.hwe_normalized_pca(mt.GT) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=10, burn_in_iterations=4) def benchmark_hwe_normalized_pca_blanczos_small_data_0_iterations(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt = mt.filter_rows(mt.info.AF[0] > 0.01) @@ -122,13 +122,13 @@ def benchmark_hwe_normalized_pca_blanczos_small_data_10_iterations(profile25_mt) hl._hwe_normalized_blanczos(mt.GT, q_iterations=10) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=10, burn_in_iterations=10) def benchmark_split_multi_hts(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) hl.split_multi_hts(mt)._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=5, burn_in_iterations=4) def benchmark_split_multi(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) hl.split_multi(mt)._force_count_rows() @@ -143,7 +143,7 @@ def benchmark_concordance(profile25_mt): c._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=10, iterations=10, burn_in_iterations=9) def benchmark_genetics_pipeline(profile25_mt, tmp_path): mt = hl.read_matrix_table(str(profile25_mt)) mt = hl.split_multi_hts(mt) @@ -180,7 +180,7 @@ def benchmark_pc_relate_5k_5k(balding_nichols_5k_5k): rel._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=15, iterations=5, burn_in_iterations=10) def benchmark_linear_regression_rows(random_doubles_mt): mt = hl.read_matrix_table(str(random_doubles_mt)) num_phenos = 100 @@ -210,7 +210,7 @@ def benchmark_linear_regression_rows_nd(random_doubles_mt): res._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=10, burn_in_iterations=5) def benchmark_logistic_regression_rows_wald(random_doubles_mt): mt = hl.read_matrix_table(str(random_doubles_mt)) mt = mt.head(2000) diff --git a/hail/python/benchmark/hail/benchmark_sentinel.py b/hail/python/benchmark/hail/benchmark_sentinel.py index 8f82fc648406..71a2bff3c577 100644 --- a/hail/python/benchmark/hail/benchmark_sentinel.py +++ b/hail/python/benchmark/hail/benchmark_sentinel.py @@ -3,14 +3,14 @@ import pytest -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=15, iterations=10, burn_in_iterations=10) def benchmark_sentinel_read_gunzip(many_ints_tsv): with gzip.open(many_ints_tsv) as f: for _ in f: pass -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=5, burn_in_iterations=5) def benchmark_sentinel_cpu_hash_1(): x = 0 for _ in range(10_000): diff --git a/hail/python/benchmark/hail/benchmark_shuffle.py b/hail/python/benchmark/hail/benchmark_shuffle.py index d1c963feeff6..5be30fc13be3 100644 --- a/hail/python/benchmark/hail/benchmark_shuffle.py +++ b/hail/python/benchmark/hail/benchmark_shuffle.py @@ -17,7 +17,7 @@ def new_query_tmpdir(tmp_path): backend.local_tmpdir = old -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=10, iterations=10, burn_in_iterations=10) def benchmark_shuffle_key_rows_by_mt(profile25_mt): mt = hl.read_matrix_table(str(profile25_mt)) mt = mt.annotate_rows(reversed_position_locus=hl.struct(contig=mt.locus.contig, position=-mt.locus.position)) @@ -25,14 +25,14 @@ def benchmark_shuffle_key_rows_by_mt(profile25_mt): mt._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=10, iterations=10, burn_in_iterations=22) def benchmark_shuffle_order_by_10m_int(): t = hl.utils.range_table(10_000_000, n_partitions=100) t = t.order_by(-t.idx) t._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=10, burn_in_iterations=4) def benchmark_shuffle_key_rows_by_4096_byte_rows(): mt = hl.utils.range_matrix_table(100_000, (1 << 12) // 4) mt = mt.annotate_entries(entry=mt.row_idx * mt.col_idx) @@ -40,7 +40,7 @@ def benchmark_shuffle_key_rows_by_4096_byte_rows(): mt._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=10, burn_in_iterations=4) def benchmark_shuffle_key_rows_by_65k_byte_rows(): mt = hl.utils.range_matrix_table(10_000, (1 << 16) // 4) mt = mt.annotate_entries(entry=mt.row_idx * mt.col_idx) @@ -48,13 +48,13 @@ def benchmark_shuffle_key_rows_by_65k_byte_rows(): mt._force_count_rows() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=15, iterations=5, burn_in_iterations=8) def benchmark_shuffle_key_by_aggregate_bad_locality(many_ints_ht): ht = hl.read_table(str(many_ints_ht)) ht.group_by(x=ht.i0 % 1000).aggregate(c=hl.agg.count(), m=hl.agg.mean(ht.i2))._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=15, iterations=10, burn_in_iterations=5) def benchmark_shuffle_key_by_aggregate_good_locality(many_ints_ht): ht = hl.read_table(str(many_ints_ht)) divisor = 7_500_000 / 51 # should ensure each partition never overflows default buffer size diff --git a/hail/python/benchmark/hail/benchmark_table.py b/hail/python/benchmark/hail/benchmark_table.py index 295e4701b002..ec5ebc6a0c11 100644 --- a/hail/python/benchmark/hail/benchmark_table.py +++ b/hail/python/benchmark/hail/benchmark_table.py @@ -253,7 +253,7 @@ def benchmark_table_expr_take(many_strings_ht): hl.tuple([ht.f1, ht.f2]).take(100) -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=15, iterations=10, burn_in_iterations=10) def benchmark_read_force_count_partitions(many_partitions_ht): hl.read_table(str(many_partitions_ht))._force_count() @@ -286,20 +286,20 @@ def benchmark_union_partitions_table(many_partitions_ht1, many_partitions_ht2): ht1.union(ht2)._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=15, burn_in_iterations=14) def benchmark_join_partitions_table(many_partitions_ht1, many_partitions_ht2): ht1 = hl.read_table(str(many_partitions_ht1)) ht2 = hl.read_table(str(many_partitions_ht2)) ht1.join(ht2)._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=20, iterations=15, burn_in_iterations=8) def benchmark_group_by_collect_per_row(gnomad_dp_sim): ht = hl.read_matrix_table(str(gnomad_dp_sim)).localize_entries('e', 'c') ht.group_by(*ht.key).aggregate(value=hl.agg.collect(ht.row_value))._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.2, instances=10, iterations=5, burn_in_iterations=10) def benchmark_group_by_take_rekey(gnomad_dp_sim): ht = hl.read_matrix_table(str(gnomad_dp_sim)).localize_entries('e', 'c') ht.group_by(k=hl.int(ht.row_idx / 50)).aggregate(value=hl.agg.take(ht.row_value, 1))._force_count() @@ -342,7 +342,7 @@ def benchmark_test_inner_join_region_memory(): joined._force_count() -@pytest.mark.benchmark() +@pytest.mark.benchmark(mds=1.1, instances=10, iterations=10, burn_in_iterations=9) def benchmark_test_left_join_region_memory(): high_mem_table = hl.utils.range_table(30).naive_coalesce(1).annotate(big_array=hl.zeros(50_000_000)) high_mem_table2 = hl.utils.range_table(30).naive_coalesce(1).annotate(big_array=hl.zeros(50_000_000))