From 48efe26f8d8ae209c1afcc69bac92d3357bade2e Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Sat, 20 May 2023 14:00:44 +0200 Subject: [PATCH 01/40] update codes --- alphapulldown/run_multimer_jobs.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/alphapulldown/run_multimer_jobs.py b/alphapulldown/run_multimer_jobs.py index 738773da..8df7701f 100755 --- a/alphapulldown/run_multimer_jobs.py +++ b/alphapulldown/run_multimer_jobs.py @@ -305,6 +305,33 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ if not isinstance(multimer_object, MultimericObject): multimer_object.input_seqs = [multimer_object.sequence] + if not use_unifold: + predict( + model_runners, + output_path, + multimer_object.feature_dict, + random_seed, + FLAGS.benchmark, + fasta_name=multimer_object.description, + models_to_relax=FLAGS.models_to_relax, + seqs=multimer_object.input_seqs, + ) + create_and_save_pae_plots(multimer_object, output_path) + else: + from unifold.inference import config_args,unifold_config_model,unifold_predict + from unifold.dataset import process + from unifold.config import model_config + configs = model_config("multimer_af2_v3") + general_args = config_args(FLAGS.unifold_param,target_name=multimer_object.description,output_dir=output_path) + model_runner = unifold_config_model(general_args) + # First need to add num_recycling_iters to the feature dictionary + # multimer_object.feature_dict.update({"num_recycling_iters":general_args.max_recycling_iters}) + processed_features,_ = process(config=configs.data,features=multimer_object.feature_dict, + mode="predict",labels=None, + seed=42,batch_idx=None, + data_idx=None,is_distillation=False) + logging.info(f"finished configuring the Unifold AlphlaFold model and process numpy features") + unifold_predict(model_runner,general_args,processed_features) predict( model_runners, From 2a263162c714cc0f8d982a0455a60eb10ad799a5 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Wed, 24 May 2023 15:16:37 +0200 Subject: [PATCH 02/40] now make the model structure of unifold flexible with a flag --- alphapulldown/run_multimer_jobs.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/alphapulldown/run_multimer_jobs.py b/alphapulldown/run_multimer_jobs.py index 8df7701f..b51f1def 100755 --- a/alphapulldown/run_multimer_jobs.py +++ b/alphapulldown/run_multimer_jobs.py @@ -67,6 +67,8 @@ "msa_depth", None, "Number of sequences to use from the MSA (by default is taken from AF model config)" ) +flags.DEFINE_enum("unifold_model_name","multimer_af2", + ["multimer_af2","multimer_ft","multimer","multimer_af2_v3","multimer_af2_model45_v3"]) flags.mark_flag_as_required("output_path") delattr(flags.FLAGS, "models_to_relax") @@ -319,14 +321,14 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ create_and_save_pae_plots(multimer_object, output_path) else: from unifold.inference import config_args,unifold_config_model,unifold_predict - from unifold.dataset import process + from unifold.dataset import process_ap from unifold.config import model_config - configs = model_config("multimer_af2_v3") + configs = model_config(FLAGS.unifold_model_name) general_args = config_args(FLAGS.unifold_param,target_name=multimer_object.description,output_dir=output_path) model_runner = unifold_config_model(general_args) # First need to add num_recycling_iters to the feature dictionary # multimer_object.feature_dict.update({"num_recycling_iters":general_args.max_recycling_iters}) - processed_features,_ = process(config=configs.data,features=multimer_object.feature_dict, + processed_features,_ = process_ap(config=configs.data,features=multimer_object.feature_dict, mode="predict",labels=None, seed=42,batch_idx=None, data_idx=None,is_distillation=False) From a8eec285e8c61d08790663b9ae8dda44a0553af0 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 25 May 2023 13:43:28 +0200 Subject: [PATCH 03/40] fixed the flags error --- alphapulldown/run_multimer_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alphapulldown/run_multimer_jobs.py b/alphapulldown/run_multimer_jobs.py index b51f1def..467fd950 100755 --- a/alphapulldown/run_multimer_jobs.py +++ b/alphapulldown/run_multimer_jobs.py @@ -68,7 +68,7 @@ ) flags.DEFINE_enum("unifold_model_name","multimer_af2", - ["multimer_af2","multimer_ft","multimer","multimer_af2_v3","multimer_af2_model45_v3"]) + ["multimer_af2","multimer_ft","multimer","multimer_af2_v3","multimer_af2_model45_v3"],"choose unifold model structure") flags.mark_flag_as_required("output_path") delattr(flags.FLAGS, "models_to_relax") From 53b5a76f1a3d3d6975ef59d1b1836e8af7c25f59 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Fri, 13 Oct 2023 21:43:23 +0200 Subject: [PATCH 04/40] update gitmodules with unifold submodule --- .gitmodules | 4 ++++ unifold | 1 + 2 files changed, 5 insertions(+) create mode 160000 unifold diff --git a/.gitmodules b/.gitmodules index f9a8013e..62946fa0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,7 @@ path = alphapulldown/analysis_pipeline/af2plots url = https://gitlab.com/gchojnowski/af2plots.git branch = main +[submodule "unifold"] + path = unifold + url = https://github.com/dingquanyu/Uni-Fold.git + branch = main diff --git a/unifold b/unifold new file mode 160000 index 00000000..abdbee59 --- /dev/null +++ b/unifold @@ -0,0 +1 @@ +Subproject commit abdbee59651738c84907611a8f214c29e8bc272e From 38e5f1c3e07e81c9dc770e7be67e4ade37ecd5cf Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Fri, 13 Oct 2023 21:44:12 +0200 Subject: [PATCH 05/40] start working on adding crosslink input --- alphapulldown/run_multimer_jobs.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/alphapulldown/run_multimer_jobs.py b/alphapulldown/run_multimer_jobs.py index 467fd950..e3625c42 100755 --- a/alphapulldown/run_multimer_jobs.py +++ b/alphapulldown/run_multimer_jobs.py @@ -66,7 +66,20 @@ flags.DEFINE_integer( "msa_depth", None, "Number of sequences to use from the MSA (by default is taken from AF model config)" ) +flags.DEFINE_boolean( + "use_unifold",False,"Whether unifold models are going to be used. Default it False" +) + +flags.DEFINE_boolean( + "use_alphalink",False,"Whether alphalink models are going to be used. Default it False" +) +flags.DEFINE_string( + "alphalink_weight",None,'Path to AlphaLink neural network weights' +) +flags.DEFINE_string( + "unifold_param",None,'Path to UniFold neural network weights' +) flags.DEFINE_enum("unifold_model_name","multimer_af2", ["multimer_af2","multimer_ft","multimer","multimer_af2_v3","multimer_af2_model45_v3"],"choose unifold model structure") flags.mark_flag_as_required("output_path") @@ -307,7 +320,7 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ if not isinstance(multimer_object, MultimericObject): multimer_object.input_seqs = [multimer_object.sequence] - if not use_unifold: + if not FLAGS.use_unifold: predict( model_runners, output_path, @@ -328,10 +341,12 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ model_runner = unifold_config_model(general_args) # First need to add num_recycling_iters to the feature dictionary # multimer_object.feature_dict.update({"num_recycling_iters":general_args.max_recycling_iters}) - processed_features,_ = process_ap(config=configs.data,features=multimer_object.feature_dict, - mode="predict",labels=None, - seed=42,batch_idx=None, - data_idx=None,is_distillation=False) + processed_features,_ = process_ap(config=configs.data, + features=multimer_object.feature_dict, + mode="predict",labels=None, + seed=42,batch_idx=None, + data_idx=None,is_distillation=False, + chain_id_map = multimer_object.chain_id_map) logging.info(f"finished configuring the Unifold AlphlaFold model and process numpy features") unifold_predict(model_runner,general_args,processed_features) From 39a60a35a338e9d2763c28a10ac75b552b9dffdb Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Fri, 13 Oct 2023 21:45:07 +0200 Subject: [PATCH 06/40] added unifold submodule codes --- unifold | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unifold b/unifold index abdbee59..e6a85f4b 160000 --- a/unifold +++ b/unifold @@ -1 +1 @@ -Subproject commit abdbee59651738c84907611a8f214c29e8bc272e +Subproject commit e6a85f4b5f5adaf20922c852f30f8dd686d370eb From 5e98ed889805ae357389d51016432c492dbeabf8 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Fri, 13 Oct 2023 21:49:21 +0200 Subject: [PATCH 07/40] added unittest for preparing crosslink infor --- test/test_crosslink_input.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 test/test_crosslink_input.py diff --git a/test/test_crosslink_input.py b/test/test_crosslink_input.py new file mode 100644 index 00000000..d531f535 --- /dev/null +++ b/test/test_crosslink_input.py @@ -0,0 +1,13 @@ +import unittest +from unifold.datset import process_xl_input +from alphafold.data.pipeline_multimer import _FastaChain +class TestCreateObjects(unittest.TestCase): + def setUp(self) -> None: + self.crosslink_info ="./test_data/test_crosslink_info.pkl.gz" + self.asym_id = [1]*10 + [2]*25 + [3]*40 + self.chain_id_map = { + "A":_FastaChain(sequence='',description='chain1'), + "B":_FastaChain(sequence='',description='chain2'), + "C":_FastaChain(sequence='',description='chain3') + } + return super().setUp() \ No newline at end of file From a69c7759f23b4300072197ee306fcd3f7135d46e Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Sun, 15 Oct 2023 14:29:50 +0200 Subject: [PATCH 08/40] update test of process_xl_input functions --- test/test_crosslink_input.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/test/test_crosslink_input.py b/test/test_crosslink_input.py index d531f535..ce231990 100644 --- a/test/test_crosslink_input.py +++ b/test/test_crosslink_input.py @@ -1,13 +1,37 @@ import unittest -from unifold.datset import process_xl_input +from unifold.dataset import process_xl_input,calculate_offsets,create_xl_features from alphafold.data.pipeline_multimer import _FastaChain +import numpy as np +import gzip,pickle +import torch class TestCreateObjects(unittest.TestCase): def setUp(self) -> None: - self.crosslink_info ="./test_data/test_crosslink_info.pkl.gz" + self.crosslink_info ="./test/test_data/test_xl_input.pkl.gz" self.asym_id = [1]*10 + [2]*25 + [3]*40 self.chain_id_map = { "A":_FastaChain(sequence='',description='chain1'), "B":_FastaChain(sequence='',description='chain2'), "C":_FastaChain(sequence='',description='chain3') } - return super().setUp() \ No newline at end of file + return super().setUp() + + def test1_calculate_offsets(self): + offsets = calculate_offsets(self.asym_id) + offsets = offsets.tolist() + expected_offsets = [0,10,35,75] + self.assertEqual(offsets,expected_offsets) + + def test2_create_xl_inputs(self): + offsets = calculate_offsets(self.asym_id) + xl_pickle = pickle.load(gzip.open(self.crosslink_info,'rb')) + xl = create_xl_features(xl_pickle,offsets,chain_id_map = self.chain_id_map) + expected_xl = torch.tensor([[10,35,0.01], + [3,27,0.01], + [5,56,0.01], + [20,65,0.01]]) + self.assertTrue(torch.equal(xl,expected_xl)) + + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From eebac8363663a669a66f6be26228ee308037eee9 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Sun, 15 Oct 2023 15:08:51 +0200 Subject: [PATCH 09/40] finished testing xl input preparation --- test/test_crosslink_input.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/test_crosslink_input.py b/test/test_crosslink_input.py index ce231990..de31716b 100644 --- a/test/test_crosslink_input.py +++ b/test/test_crosslink_input.py @@ -1,5 +1,5 @@ import unittest -from unifold.dataset import process_xl_input,calculate_offsets,create_xl_features +from unifold.dataset import calculate_offsets,create_xl_features,bin_xl from alphafold.data.pipeline_multimer import _FastaChain import numpy as np import gzip,pickle @@ -13,6 +13,7 @@ def setUp(self) -> None: "B":_FastaChain(sequence='',description='chain2'), "C":_FastaChain(sequence='',description='chain3') } + self.bins = torch.arange(0,1.05,0.05) return super().setUp() def test1_calculate_offsets(self): @@ -31,7 +32,18 @@ def test2_create_xl_inputs(self): [20,65,0.01]]) self.assertTrue(torch.equal(xl,expected_xl)) - + def test3_bin_xl(self): + offsets = calculate_offsets(self.asym_id) + xl_pickle = pickle.load(gzip.open(self.crosslink_info,'rb')) + xl = create_xl_features(xl_pickle,offsets,chain_id_map = self.chain_id_map) + num_res = len(self.asym_id) + xl = bin_xl(xl,num_res) + expected_xl = np.zeros((num_res,num_res,1)) + expected_xl[3,27,0] = expected_xl[27,3,0] = torch.bucketize(0.99,self.bins) + expected_xl[10,35,0] = expected_xl[35,10,0] = torch.bucketize(0.99,self.bins) + expected_xl[5,56,0] = expected_xl[56,5,0] = torch.bucketize(0.99,self.bins) + expected_xl[20,65,0] = expected_xl[65,20,0] = torch.bucketize(0.99,self.bins) + self.assertTrue(np.array_equal(xl,expected_xl)) if __name__ == "__main__": unittest.main() \ No newline at end of file From 4a33322a0fc0d68cd85ac4bccd7c4f8a52b8d98f Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Sun, 15 Oct 2023 15:41:34 +0200 Subject: [PATCH 10/40] start working on alphalink inference codes --- alphapulldown/run_multimer_jobs.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/alphapulldown/run_multimer_jobs.py b/alphapulldown/run_multimer_jobs.py index e3625c42..281262c4 100755 --- a/alphapulldown/run_multimer_jobs.py +++ b/alphapulldown/run_multimer_jobs.py @@ -73,7 +73,9 @@ flags.DEFINE_boolean( "use_alphalink",False,"Whether alphalink models are going to be used. Default it False" ) - +flags.DEFINE_string( + "crosslinks",None,"Path to crosslink information pickle" +) flags.DEFINE_string( "alphalink_weight",None,'Path to AlphaLink neural network weights' ) @@ -332,7 +334,7 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ seqs=multimer_object.input_seqs, ) create_and_save_pae_plots(multimer_object, output_path) - else: + elif FLAGS.use_unifold: from unifold.inference import config_args,unifold_config_model,unifold_predict from unifold.dataset import process_ap from unifold.config import model_config @@ -346,9 +348,13 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ mode="predict",labels=None, seed=42,batch_idx=None, data_idx=None,is_distillation=False, - chain_id_map = multimer_object.chain_id_map) + chain_id_map = multimer_object.chain_id_map, + ) logging.info(f"finished configuring the Unifold AlphlaFold model and process numpy features") unifold_predict(model_runner,general_args,processed_features) + elif FLAGS.use_alphalink: + assert FLAGS.crosslinks is not None + assert FLAGS.alphalink_weight is not None predict( model_runners, From 2811bf4b86ed654e9564717987ba7c3f3d7eb27e Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Sun, 15 Oct 2023 17:45:55 +0200 Subject: [PATCH 11/40] finished adding alphalink inference codes --- alphapulldown/run_multimer_jobs.py | 34 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/alphapulldown/run_multimer_jobs.py b/alphapulldown/run_multimer_jobs.py index 281262c4..6fa69a0a 100755 --- a/alphapulldown/run_multimer_jobs.py +++ b/alphapulldown/run_multimer_jobs.py @@ -339,7 +339,9 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ from unifold.dataset import process_ap from unifold.config import model_config configs = model_config(FLAGS.unifold_model_name) - general_args = config_args(FLAGS.unifold_param,target_name=multimer_object.description,output_dir=output_path) + general_args = config_args(FLAGS.unifold_param, + target_name=multimer_object.description, + output_dir=output_path) model_runner = unifold_config_model(general_args) # First need to add num_recycling_iters to the feature dictionary # multimer_object.feature_dict.update({"num_recycling_iters":general_args.max_recycling_iters}) @@ -347,28 +349,26 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ features=multimer_object.feature_dict, mode="predict",labels=None, seed=42,batch_idx=None, - data_idx=None,is_distillation=False, - chain_id_map = multimer_object.chain_id_map, + data_idx=None,is_distillation=False ) logging.info(f"finished configuring the Unifold AlphlaFold model and process numpy features") unifold_predict(model_runner,general_args,processed_features) + elif FLAGS.use_alphalink: assert FLAGS.crosslinks is not None assert FLAGS.alphalink_weight is not None - - predict( - model_runners, - output_path, - multimer_object.feature_dict, - random_seed, - FLAGS.benchmark, - fasta_name=multimer_object.description, - models_to_relax=FLAGS.models_to_relax, - seqs=multimer_object.input_seqs, - ) - create_and_save_pae_plots(multimer_object, output_path) - - + from unifold.alphalink_inference import alphalink_prediction + from unifold.dataset import process_ap + processed_features,_ = process_ap(config=configs.data, + features=multimer_object.feature_dict, + mode="predict",labels=None, + seed=42,batch_idx=None, + data_idx=None,is_distillation=False, + chain_id_map = multimer_object.chain_id_map, + crosslinks = FLAGS.crosslinks + ) + alphalink_prediction(processed_features,FLAGS.output_path, + param_path = FLAGS.alphalink_weight) def predict_multimers(multimers): From 855619c38f0b598ff5990732f62e179fa70e92ed Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 26 Oct 2023 14:35:18 +0200 Subject: [PATCH 12/40] update run_multimer_jobs --- alphapulldown/run_multimer_jobs.py | 34 +++++++++++++++++------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/alphapulldown/run_multimer_jobs.py b/alphapulldown/run_multimer_jobs.py index 6fa69a0a..bda8b411 100755 --- a/alphapulldown/run_multimer_jobs.py +++ b/alphapulldown/run_multimer_jobs.py @@ -322,19 +322,7 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ if not isinstance(multimer_object, MultimericObject): multimer_object.input_seqs = [multimer_object.sequence] - if not FLAGS.use_unifold: - predict( - model_runners, - output_path, - multimer_object.feature_dict, - random_seed, - FLAGS.benchmark, - fasta_name=multimer_object.description, - models_to_relax=FLAGS.models_to_relax, - seqs=multimer_object.input_seqs, - ) - create_and_save_pae_plots(multimer_object, output_path) - elif FLAGS.use_unifold: + if FLAGS.use_unifold: from unifold.inference import config_args,unifold_config_model,unifold_predict from unifold.dataset import process_ap from unifold.config import model_config @@ -353,12 +341,16 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ ) logging.info(f"finished configuring the Unifold AlphlaFold model and process numpy features") unifold_predict(model_runner,general_args,processed_features) - + elif FLAGS.use_alphalink: assert FLAGS.crosslinks is not None assert FLAGS.alphalink_weight is not None from unifold.alphalink_inference import alphalink_prediction from unifold.dataset import process_ap + from unifold.config import model_config + logging.info(f"###### Start using AlphaLink weights and cross-link information") + MODEL_NAME = 'model_5_ptm_af2' + configs = model_config(MODEL_NAME) processed_features,_ = process_ap(config=configs.data, features=multimer_object.feature_dict, mode="predict",labels=None, @@ -366,9 +358,21 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ data_idx=None,is_distillation=False, chain_id_map = multimer_object.chain_id_map, crosslinks = FLAGS.crosslinks - ) + ) alphalink_prediction(processed_features,FLAGS.output_path, param_path = FLAGS.alphalink_weight) + else: + predict( + model_runners, + output_path, + multimer_object.feature_dict, + random_seed, + FLAGS.benchmark, + fasta_name=multimer_object.description, + models_to_relax=FLAGS.models_to_relax, + seqs=multimer_object.input_seqs, + ) + create_and_save_pae_plots(multimer_object, output_path) def predict_multimers(multimers): From 572f9ccd83303705a55269685ac411962ea52475 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 26 Oct 2023 14:38:41 +0200 Subject: [PATCH 13/40] update unifold submodule --- unifold | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unifold b/unifold index e6a85f4b..5acbad09 160000 --- a/unifold +++ b/unifold @@ -1 +1 @@ -Subproject commit e6a85f4b5f5adaf20922c852f30f8dd686d370eb +Subproject commit 5acbad0905b412f64b2bdaf5882aa68bd14d2b99 From f0dc1998ac25cc85d833639db9d37189719bac1d Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 26 Oct 2023 17:23:39 +0200 Subject: [PATCH 14/40] unittest for cross link input and inference with crosslink data --- test/test_crosslink_inference.py | 65 ++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 test/test_crosslink_inference.py diff --git a/test/test_crosslink_inference.py b/test/test_crosslink_inference.py new file mode 100644 index 00000000..22623ef9 --- /dev/null +++ b/test/test_crosslink_inference.py @@ -0,0 +1,65 @@ +import shutil +import tempfile +import unittest +import sys +import os +import torch +from unifold.modules.alphafold import AlphaFold +from unifold.alphalink_inference import prepare_model_runner +from unifold.alphalink_inference import alphalink_prediction +from unifold.dataset import process_ap +from unifold.config import model_config +from alphapulldown.utils import create +from alphapulldown.run_multimer_jobs import predict_individual_jobs,create_custom_jobs + +class _TestBase(unittest.TestCase): + def setUp(self) -> None: + self.crosslink_file_path = os.path.join(os.path.dirname(__file__),"test_data/example_crosslink.pkl.gz") + self.config_data_model_name = 'model_5_ptm_af2' + self.config_alphafold_model_name = 'multimer_af2_crop' + +class TestCrosslinkInference(_TestBase): + def setUp(self) -> None: + super().setUp() + self.output_dir = tempfile.mkdtemp() + self.monomer_object_path = os.path.join(os.path.dirname(__file__),"test_data/") + self.protein_list = os.path.join(os.path.dirname(__file__),"test_data/example_crosslinked_pair.txt") + self.alphalink_weight = '/g/alphafold/alphalink_weights/AlphaLink-Multimer_SDA_v3.pt' + self.multimerobject = create_custom_jobs(self.protein_list,self.monomer_object_path,job_index=1,pair_msa=True)[0] + + def test1_process_features(self): + """Test whether the PyTorch model of AlphaLink can be initiated successfully""" + configs = model_config(self.config_data_model_name) + processed_features,_ = process_ap(config=configs.data, + features=self.multimerobject.feature_dict, + mode="predict",labels=None, + seed=42,batch_idx=None, + data_idx=None,is_distillation=False, + chain_id_map = self.multimerobject.chain_id_map, + crosslinks = self.crosslink_file_path + ) + + def test2_load_AlphaLink_weights(self): + """This is testing weither loading the PyTorch checkpoint is sucessfull""" + if torch.cuda.is_available(): + model_device = 'cuda:0' + else: + model_device = 'cpu' + + config = model_config(self.config_alphafold_model_name) + model = AlphaFold(config) + state_dict = torch.load(self.alphalink_weight)["ema"]["params"] + state_dict = {".".join(k.split(".")[1:]): v for k, v in state_dict.items()} + model.load_state_dict(state_dict) + model.to(model_device) + + def test3_test_inference(self): + if torch.cuda.is_available(): + model_device = 'cuda:0' + else: + model_device = 'cpu' + model = prepare_model_runner(self.alphalink_weight,model_device=model_device) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 55608b12f60d715d7e1c5ef9978156651640c9c3 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Wed, 1 Nov 2023 14:48:11 +0100 Subject: [PATCH 15/40] update alphafold, unifold submodules --- alphafold | 2 +- unifold | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/alphafold b/alphafold index d13534bd..780ffa4d 160000 --- a/alphafold +++ b/alphafold @@ -1 +1 @@ -Subproject commit d13534bd45754ded6190b1b4aa9a00dab52ed75c +Subproject commit 780ffa4db323cf4382f77a04d496c8b6043337a8 diff --git a/unifold b/unifold index 5acbad09..3186dee1 160000 --- a/unifold +++ b/unifold @@ -1 +1 @@ -Subproject commit 5acbad0905b412f64b2bdaf5882aa68bd14d2b99 +Subproject commit 3186dee1a51a8fcbb8209cd3cee6b15086fe0629 From fbafbd6f34ee74f9048dbc93e7cd43fc114a54d9 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Wed, 1 Nov 2023 15:00:01 +0100 Subject: [PATCH 16/40] remove unnecessary debug statements --- alphapulldown/run_multimer_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alphapulldown/run_multimer_jobs.py b/alphapulldown/run_multimer_jobs.py index bda8b411..1b328289 100755 --- a/alphapulldown/run_multimer_jobs.py +++ b/alphapulldown/run_multimer_jobs.py @@ -348,7 +348,7 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ from unifold.alphalink_inference import alphalink_prediction from unifold.dataset import process_ap from unifold.config import model_config - logging.info(f"###### Start using AlphaLink weights and cross-link information") + logging.info(f"Start using AlphaLink weights and cross-link information") MODEL_NAME = 'model_5_ptm_af2' configs = model_config(MODEL_NAME) processed_features,_ = process_ap(config=configs.data, From 78c012de84220a47b0106ba2bbf25e0607526348 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Wed, 1 Nov 2023 15:00:53 +0100 Subject: [PATCH 17/40] update unifold submodule --- unifold | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unifold b/unifold index 3186dee1..8ccb0e70 160000 --- a/unifold +++ b/unifold @@ -1 +1 @@ -Subproject commit 3186dee1a51a8fcbb8209cd3cee6b15086fe0629 +Subproject commit 8ccb0e70885ba3b34dccb9d7d7b3e7390ac386b5 From 15572fce2cbb481d2904efaa3b55be41b2eb6f3c Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Wed, 1 Nov 2023 15:31:18 +0100 Subject: [PATCH 18/40] update github actions --- .github/workflows/github_actions.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index fad77f4b..c2bb110d 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -35,11 +35,13 @@ jobs: conda develop alphapulldown conda develop alphapulldown/ColabFold conda develop alphafold - #conda info - #conda list - #python -c "import sys; print(sys.path)" - #echo "Python path: $PYTHONPATH" + conda develop unifold python -c "import alphafold; import os; print('Alphafold module is located at:', alphafold.__file__); alphafold_dir = os.path.dirname(alphafold.__file__); print('Contents of the Alphafold directory:', os.listdir(alphafold_dir))" + - name: Check imports of submodules + run : | + python -c "import unifold; print('unifold path is :',unifold.__file__)" + python -c "from unifold.dataset import process_ap" + python -c "from unifold.inference import config_args,unifold_config_model,unifold_predict" - name: Install Dependencies run: | eval "$(conda shell.bash hook)" @@ -47,12 +49,16 @@ jobs: conda install -c omnia -c bioconda -c conda-forge openmm==8.0 pdbfixer==1.9 kalign2 cctbx-base importlib_metadata #conda install -c bioconda hmmer hhsuite #this causes conda conflicts pip install jax==0.3.25 jaxlib==0.3.25+cuda11.cudnn805 -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html + # install PyTorch + pip install torch --pre -f https://download.pytorch.org/whl/nightly/cu121/torch_nightly.html pip install "absl-py>=0.13.0" dm-haiku "dm-tree>=0.1.6" "h5py>=3.1.0" "ml-collections>=0.1.0" "pandas>=1.3.4" tensorflow "importlib-resources==5.8.0" "nbformat==5.4.0" "py3Dmol==2.0.1" ipython appdirs jupyterlab ipywidgets pytest - name: Run Tests run: | eval "$(conda shell.bash hook)" conda activate AlphaPulldown - pytest test/ + pytest -s test/test_custom_db.py + pytest -s test/test_remove_clashes_low_plddt.py + python3 -m unittest test/test_crosslink_input.py - name: Build and Push to Docker Hub uses: mr-smithers-excellent/docker-build-push@v5 with: From 479b7d1ebe1fd566c062e70c6b49d62d9398f83e Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Wed, 1 Nov 2023 15:33:42 +0100 Subject: [PATCH 19/40] update github actions --- .github/workflows/github_actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index c2bb110d..e2043434 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -38,7 +38,7 @@ jobs: conda develop unifold python -c "import alphafold; import os; print('Alphafold module is located at:', alphafold.__file__); alphafold_dir = os.path.dirname(alphafold.__file__); print('Contents of the Alphafold directory:', os.listdir(alphafold_dir))" - name: Check imports of submodules - run : | + run : | python -c "import unifold; print('unifold path is :',unifold.__file__)" python -c "from unifold.dataset import process_ap" python -c "from unifold.inference import config_args,unifold_config_model,unifold_predict" From 636157f814609dbcf0cf7ba82ec317413c46f333 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Wed, 1 Nov 2023 15:40:13 +0100 Subject: [PATCH 20/40] update github actions --- .github/workflows/github_actions.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index e2043434..c1eaff6b 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -39,9 +39,10 @@ jobs: python -c "import alphafold; import os; print('Alphafold module is located at:', alphafold.__file__); alphafold_dir = os.path.dirname(alphafold.__file__); print('Contents of the Alphafold directory:', os.listdir(alphafold_dir))" - name: Check imports of submodules run : | - python -c "import unifold; print('unifold path is :',unifold.__file__)" + conda activate AlphaPulldown python -c "from unifold.dataset import process_ap" python -c "from unifold.inference import config_args,unifold_config_model,unifold_predict" + python -c "from colabfold.batch import get_queries, unserialize_msa, get_msa_and_templates, msa_to_str, build_monomer_feature, parse_fasta" - name: Install Dependencies run: | eval "$(conda shell.bash hook)" From bda5afff8a300bbf522717fcda4e4d1ba7285250 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Wed, 1 Nov 2023 15:46:58 +0100 Subject: [PATCH 21/40] added conda shell.bash hook before testing imports --- .github/workflows/github_actions.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index c1eaff6b..32c7fa55 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -39,6 +39,7 @@ jobs: python -c "import alphafold; import os; print('Alphafold module is located at:', alphafold.__file__); alphafold_dir = os.path.dirname(alphafold.__file__); print('Contents of the Alphafold directory:', os.listdir(alphafold_dir))" - name: Check imports of submodules run : | + eval "$(conda shell.bash hook)" conda activate AlphaPulldown python -c "from unifold.dataset import process_ap" python -c "from unifold.inference import config_args,unifold_config_model,unifold_predict" From beafd98343fda202e296ff5dcac117b820d29017 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Wed, 1 Nov 2023 15:52:39 +0100 Subject: [PATCH 22/40] update github actions --- .github/workflows/github_actions.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index 32c7fa55..7d9e798c 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -35,7 +35,8 @@ jobs: conda develop alphapulldown conda develop alphapulldown/ColabFold conda develop alphafold - conda develop unifold + cd unifold + python3 setup.py install python -c "import alphafold; import os; print('Alphafold module is located at:', alphafold.__file__); alphafold_dir = os.path.dirname(alphafold.__file__); print('Contents of the Alphafold directory:', os.listdir(alphafold_dir))" - name: Check imports of submodules run : | From 8bd7f5a726b5c04ea237b99bd8f22cd421c536c1 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Wed, 1 Nov 2023 15:56:26 +0100 Subject: [PATCH 23/40] update github actions --- .github/workflows/github_actions.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index 7d9e798c..65f7413c 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -36,12 +36,14 @@ jobs: conda develop alphapulldown/ColabFold conda develop alphafold cd unifold - python3 setup.py install + python3 setup.py install python -c "import alphafold; import os; print('Alphafold module is located at:', alphafold.__file__); alphafold_dir = os.path.dirname(alphafold.__file__); print('Contents of the Alphafold directory:', os.listdir(alphafold_dir))" - name: Check imports of submodules run : | eval "$(conda shell.bash hook)" conda activate AlphaPulldown + # install PyTorch + pip install torch --pre -f https://download.pytorch.org/whl/nightly/cu121/torch_nightly.html python -c "from unifold.dataset import process_ap" python -c "from unifold.inference import config_args,unifold_config_model,unifold_predict" python -c "from colabfold.batch import get_queries, unserialize_msa, get_msa_and_templates, msa_to_str, build_monomer_feature, parse_fasta" @@ -52,8 +54,6 @@ jobs: conda install -c omnia -c bioconda -c conda-forge openmm==8.0 pdbfixer==1.9 kalign2 cctbx-base importlib_metadata #conda install -c bioconda hmmer hhsuite #this causes conda conflicts pip install jax==0.3.25 jaxlib==0.3.25+cuda11.cudnn805 -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html - # install PyTorch - pip install torch --pre -f https://download.pytorch.org/whl/nightly/cu121/torch_nightly.html pip install "absl-py>=0.13.0" dm-haiku "dm-tree>=0.1.6" "h5py>=3.1.0" "ml-collections>=0.1.0" "pandas>=1.3.4" tensorflow "importlib-resources==5.8.0" "nbformat==5.4.0" "py3Dmol==2.0.1" ipython appdirs jupyterlab ipywidgets pytest - name: Run Tests run: | From 7d522a7c27f588c7a05bd6833d475cb525839f59 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Wed, 1 Nov 2023 16:06:52 +0100 Subject: [PATCH 24/40] update github actions --- .github/workflows/github_actions.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index 65f7413c..38606b90 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -42,6 +42,10 @@ jobs: run : | eval "$(conda shell.bash hook)" conda activate AlphaPulldown + # setup unicore + git clone https://github.com/dptech-corp/Uni-Core.git + cd Uni-Core + python3 setup.py install --disable-cuda-ext # install PyTorch pip install torch --pre -f https://download.pytorch.org/whl/nightly/cu121/torch_nightly.html python -c "from unifold.dataset import process_ap" From ad75e32c2e7a002836306ae97469d856be4da649 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Wed, 1 Nov 2023 16:10:41 +0100 Subject: [PATCH 25/40] update github actions --- .github/workflows/github_actions.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index 38606b90..6f3b83cf 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -42,12 +42,12 @@ jobs: run : | eval "$(conda shell.bash hook)" conda activate AlphaPulldown + # install PyTorch + pip install torch --pre -f https://download.pytorch.org/whl/nightly/cu121/torch_nightly.html # setup unicore git clone https://github.com/dptech-corp/Uni-Core.git cd Uni-Core python3 setup.py install --disable-cuda-ext - # install PyTorch - pip install torch --pre -f https://download.pytorch.org/whl/nightly/cu121/torch_nightly.html python -c "from unifold.dataset import process_ap" python -c "from unifold.inference import config_args,unifold_config_model,unifold_predict" python -c "from colabfold.batch import get_queries, unserialize_msa, get_msa_and_templates, msa_to_str, build_monomer_feature, parse_fasta" From b569722e6e4f3ebc3850037cdcfac8774cc04d9b Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Wed, 1 Nov 2023 16:18:20 +0100 Subject: [PATCH 26/40] update github actions --- .github/workflows/github_actions.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index 6f3b83cf..38099e4e 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -48,8 +48,9 @@ jobs: git clone https://github.com/dptech-corp/Uni-Core.git cd Uni-Core python3 setup.py install --disable-cuda-ext + python -c "from unifold.alphalink_inference import alphalink_prediction" python -c "from unifold.dataset import process_ap" - python -c "from unifold.inference import config_args,unifold_config_model,unifold_predict" + python -c "from unifold.config import model_config" python -c "from colabfold.batch import get_queries, unserialize_msa, get_msa_and_templates, msa_to_str, build_monomer_feature, parse_fasta" - name: Install Dependencies run: | From b14baeafb80131b895707d591e62faafa1fcdd33 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 2 Nov 2023 11:54:54 +0100 Subject: [PATCH 27/40] add pip installations --- .github/workflows/github_actions.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index 38099e4e..03038325 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -35,8 +35,10 @@ jobs: conda develop alphapulldown conda develop alphapulldown/ColabFold conda develop alphafold - cd unifold - python3 setup.py install + cd alphapulldown && pip install . + cd unifold && python3 setup.py install + cd alphapulldown/ColabFold && python3 setup.py install + python -c "import tree" python -c "import alphafold; import os; print('Alphafold module is located at:', alphafold.__file__); alphafold_dir = os.path.dirname(alphafold.__file__); print('Contents of the Alphafold directory:', os.listdir(alphafold_dir))" - name: Check imports of submodules run : | From 59caffd5c6c7b83fc86ef1279ed9e1816686165d Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 2 Nov 2023 11:59:31 +0100 Subject: [PATCH 28/40] update github actions --- .github/workflows/github_actions.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index 03038325..d3494e5d 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -35,7 +35,6 @@ jobs: conda develop alphapulldown conda develop alphapulldown/ColabFold conda develop alphafold - cd alphapulldown && pip install . cd unifold && python3 setup.py install cd alphapulldown/ColabFold && python3 setup.py install python -c "import tree" From ff4fbc7b4b03c7b8a9b495761088fbc42acf1499 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 2 Nov 2023 12:04:58 +0100 Subject: [PATCH 29/40] update github actions --- .github/workflows/github_actions.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index d3494e5d..9e1c1296 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -28,6 +28,7 @@ jobs: echo "$GITHUB_WORKSPACE/alphapulldown" >> $GITHUB_PATH - name: Install AlphaPulldown without dependencies run: | + HOMEPATH=$PWD conda create --name AlphaPulldown python=${{ matrix.python-version }} eval "$(conda shell.bash hook)" conda activate AlphaPulldown @@ -35,8 +36,9 @@ jobs: conda develop alphapulldown conda develop alphapulldown/ColabFold conda develop alphafold - cd unifold && python3 setup.py install - cd alphapulldown/ColabFold && python3 setup.py install + cd $HOMEPATH/unifold && python3 setup.py install + cd $HOMEPATH/alphafold && python3 setup.py install + cd $HOMEPATH/alphapulldown/ColabFold && python3 setup.py install python -c "import tree" python -c "import alphafold; import os; print('Alphafold module is located at:', alphafold.__file__); alphafold_dir = os.path.dirname(alphafold.__file__); print('Contents of the Alphafold directory:', os.listdir(alphafold_dir))" - name: Check imports of submodules From 299c94fa00b4ee6229caf10c61244b0c1884c5c5 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 2 Nov 2023 12:19:42 +0100 Subject: [PATCH 30/40] update github actions --- .github/workflows/github_actions.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index 9e1c1296..ccf5f730 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -28,7 +28,6 @@ jobs: echo "$GITHUB_WORKSPACE/alphapulldown" >> $GITHUB_PATH - name: Install AlphaPulldown without dependencies run: | - HOMEPATH=$PWD conda create --name AlphaPulldown python=${{ matrix.python-version }} eval "$(conda shell.bash hook)" conda activate AlphaPulldown @@ -36,11 +35,21 @@ jobs: conda develop alphapulldown conda develop alphapulldown/ColabFold conda develop alphafold - cd $HOMEPATH/unifold && python3 setup.py install - cd $HOMEPATH/alphafold && python3 setup.py install + python -c "import alphafold; import os; print('Alphafold module is located at:', alphafold.__file__); alphafold_dir = os.path.dirname(alphafold.__file__); print('Contents of the Alphafold directory:', os.listdir(alphafold_dir))" + - name: Install dependencies in AlphaFold setup.py + run: | + eval "$(conda shell.bash hook)" + export WORKDIRPATH=$PWD + conda activate AlphaPulldown + cd $WORKDIRPATH/alphafold && python3 setup.py install cd $HOMEPATH/alphapulldown/ColabFold && python3 setup.py install python -c "import tree" - python -c "import alphafold; import os; print('Alphafold module is located at:', alphafold.__file__); alphafold_dir = os.path.dirname(alphafold.__file__); print('Contents of the Alphafold directory:', os.listdir(alphafold_dir))" + - name: Install dependencies in unifold setup.py + run: | + eval "$(conda shell.bash hook)" + export WORKDIRPATH=$PWD + conda activate AlphaPulldown + cd $WORKDIRPATH/unifold && python3 setup.py install - name: Check imports of submodules run : | eval "$(conda shell.bash hook)" From c8e68e3680052dabe9131b2be432b3a28074cf05 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 2 Nov 2023 12:27:17 +0100 Subject: [PATCH 31/40] update github actions --- .github/workflows/github_actions.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index ccf5f730..c4cc44a9 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -36,20 +36,21 @@ jobs: conda develop alphapulldown/ColabFold conda develop alphafold python -c "import alphafold; import os; print('Alphafold module is located at:', alphafold.__file__); alphafold_dir = os.path.dirname(alphafold.__file__); print('Contents of the Alphafold directory:', os.listdir(alphafold_dir))" - - name: Install dependencies in AlphaFold setup.py + - name: Install dependencies in unifold setup.py run: | eval "$(conda shell.bash hook)" export WORKDIRPATH=$PWD conda activate AlphaPulldown - cd $WORKDIRPATH/alphafold && python3 setup.py install - cd $HOMEPATH/alphapulldown/ColabFold && python3 setup.py install - python -c "import tree" - - name: Install dependencies in unifold setup.py + cd $WORKDIRPATH/unifold && python3 setup.py install + - name: Install dependencies in AlphaFold setup.py run: | eval "$(conda shell.bash hook)" export WORKDIRPATH=$PWD conda activate AlphaPulldown - cd $WORKDIRPATH/unifold && python3 setup.py install + pip install protobuf==4.23.0 + cd $WORKDIRPATH/alphafold && python3 setup.py install + cd $HOMEPATH/alphapulldown/ColabFold && python3 setup.py install + python -c "import tree" - name: Check imports of submodules run : | eval "$(conda shell.bash hook)" From 1024b9797e0d66d6766a2fe89db38388632320a5 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 2 Nov 2023 12:34:58 +0100 Subject: [PATCH 32/40] update github actions --- .github/workflows/github_actions.yml | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index c4cc44a9..3194c2b3 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -49,8 +49,15 @@ jobs: conda activate AlphaPulldown pip install protobuf==4.23.0 cd $WORKDIRPATH/alphafold && python3 setup.py install - cd $HOMEPATH/alphapulldown/ColabFold && python3 setup.py install python -c "import tree" + - name: Install Dependencies + run: | + eval "$(conda shell.bash hook)" + conda activate AlphaPulldown + conda install -c omnia -c bioconda -c conda-forge openmm==8.0 pdbfixer==1.9 kalign2 cctbx-base importlib_metadata + #conda install -c bioconda hmmer hhsuite #this causes conda conflicts + pip install jax==0.3.25 jaxlib==0.3.25+cuda11.cudnn805 -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html + pip install "absl-py>=0.13.0" dm-haiku "dm-tree>=0.1.6" "h5py>=3.1.0" "ml-collections>=0.1.0" "pandas>=1.3.4" tensorflow "importlib-resources==5.8.0" "nbformat==5.4.0" "py3Dmol==2.0.1" ipython appdirs jupyterlab ipywidgets pytest - name: Check imports of submodules run : | eval "$(conda shell.bash hook)" @@ -65,27 +72,10 @@ jobs: python -c "from unifold.dataset import process_ap" python -c "from unifold.config import model_config" python -c "from colabfold.batch import get_queries, unserialize_msa, get_msa_and_templates, msa_to_str, build_monomer_feature, parse_fasta" - - name: Install Dependencies - run: | - eval "$(conda shell.bash hook)" - conda activate AlphaPulldown - conda install -c omnia -c bioconda -c conda-forge openmm==8.0 pdbfixer==1.9 kalign2 cctbx-base importlib_metadata - #conda install -c bioconda hmmer hhsuite #this causes conda conflicts - pip install jax==0.3.25 jaxlib==0.3.25+cuda11.cudnn805 -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html - pip install "absl-py>=0.13.0" dm-haiku "dm-tree>=0.1.6" "h5py>=3.1.0" "ml-collections>=0.1.0" "pandas>=1.3.4" tensorflow "importlib-resources==5.8.0" "nbformat==5.4.0" "py3Dmol==2.0.1" ipython appdirs jupyterlab ipywidgets pytest - name: Run Tests run: | eval "$(conda shell.bash hook)" conda activate AlphaPulldown pytest -s test/test_custom_db.py pytest -s test/test_remove_clashes_low_plddt.py - python3 -m unittest test/test_crosslink_input.py - - name: Build and Push to Docker Hub - uses: mr-smithers-excellent/docker-build-push@v5 - with: - image: dmolodenskiy/AlphaPulldown - registry: docker.io - dockerfile: docker/Dockerfile - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - tags: latest + python3 -m unittest test/test_crosslink_input.py \ No newline at end of file From d4c6a3eb7aafd9a3756d0ba8b7804f6399adeff1 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 2 Nov 2023 12:46:33 +0100 Subject: [PATCH 33/40] update github actions --- .github/workflows/github_actions.yml | 30 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index 3194c2b3..4eb27e7f 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -58,24 +58,24 @@ jobs: #conda install -c bioconda hmmer hhsuite #this causes conda conflicts pip install jax==0.3.25 jaxlib==0.3.25+cuda11.cudnn805 -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html pip install "absl-py>=0.13.0" dm-haiku "dm-tree>=0.1.6" "h5py>=3.1.0" "ml-collections>=0.1.0" "pandas>=1.3.4" tensorflow "importlib-resources==5.8.0" "nbformat==5.4.0" "py3Dmol==2.0.1" ipython appdirs jupyterlab ipywidgets pytest - - name: Check imports of submodules - run : | - eval "$(conda shell.bash hook)" - conda activate AlphaPulldown - # install PyTorch - pip install torch --pre -f https://download.pytorch.org/whl/nightly/cu121/torch_nightly.html - # setup unicore - git clone https://github.com/dptech-corp/Uni-Core.git - cd Uni-Core - python3 setup.py install --disable-cuda-ext - python -c "from unifold.alphalink_inference import alphalink_prediction" - python -c "from unifold.dataset import process_ap" - python -c "from unifold.config import model_config" - python -c "from colabfold.batch import get_queries, unserialize_msa, get_msa_and_templates, msa_to_str, build_monomer_feature, parse_fasta" + # - name: Check imports of submodules + # run : | + # eval "$(conda shell.bash hook)" + # conda activate AlphaPulldown + # # install PyTorch + # pip install torch --pre -f https://download.pytorch.org/whl/nightly/cu121/torch_nightly.html + # # setup unicore + # git clone https://github.com/dptech-corp/Uni-Core.git + # cd Uni-Core + # python3 setup.py install --disable-cuda-ext + # python -c "from unifold.alphalink_inference import alphalink_prediction" + # python -c "from unifold.dataset import process_ap" + # python -c "from unifold.config import model_config" + # python -c "from colabfold.batch import get_queries, unserialize_msa, get_msa_and_templates, msa_to_str, build_monomer_feature, parse_fasta" - name: Run Tests run: | eval "$(conda shell.bash hook)" conda activate AlphaPulldown pytest -s test/test_custom_db.py pytest -s test/test_remove_clashes_low_plddt.py - python3 -m unittest test/test_crosslink_input.py \ No newline at end of file + # python3 -m unittest test/test_crosslink_input.py \ No newline at end of file From 4ab16993cb9cf183998be72b319b88f8f22a976a Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 2 Nov 2023 12:51:40 +0100 Subject: [PATCH 34/40] added test data for cross-link input information --- test/test_data/example_crosslink.pkl.gz | Bin 0 -> 222 bytes test/test_data/test_xl_input.pkl.gz | Bin 0 -> 115 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 test/test_data/example_crosslink.pkl.gz create mode 100644 test/test_data/test_xl_input.pkl.gz diff --git a/test/test_data/example_crosslink.pkl.gz b/test/test_data/example_crosslink.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..ac6bd7bc74b330bcfaf8fedec0b6285fe9527369 GIT binary patch literal 222 zcmV<403rV$iwFoyuNGwj|4vgeUrtX>PhVqlZ*y~OX>MzCE^upX0Bc~GYRSj|2DMXq zMDt2>3o7-J^NUjTa!YecG82o65-X?luoWjK<|GzPnLMS36|4kEr<7C{q=H14Ld~Xh zcC=3knxf&&+{0)y#m~>rTf&>go1)R6TTGCsN%$#hck%5Q&=#RcZ`W>dopciX@d-n!#K Date: Thu, 2 Nov 2023 17:27:02 +0100 Subject: [PATCH 35/40] update alphalink prediction output path --- alphapulldown/run_multimer_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alphapulldown/run_multimer_jobs.py b/alphapulldown/run_multimer_jobs.py index 1b328289..afb33f70 100755 --- a/alphapulldown/run_multimer_jobs.py +++ b/alphapulldown/run_multimer_jobs.py @@ -359,7 +359,7 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ chain_id_map = multimer_object.chain_id_map, crosslinks = FLAGS.crosslinks ) - alphalink_prediction(processed_features,FLAGS.output_path, + alphalink_prediction(processed_features,os.path.join(FLAGS.output_path,multimer_object.description), param_path = FLAGS.alphalink_weight) else: predict( From 11d960dfa7b0384fb234398a2440326ab4fe5ed6 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 2 Nov 2023 17:34:56 +0100 Subject: [PATCH 36/40] update unifold module --- unifold | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unifold b/unifold index 8ccb0e70..649de7f3 160000 --- a/unifold +++ b/unifold @@ -1 +1 @@ -Subproject commit 8ccb0e70885ba3b34dccb9d7d7b3e7390ac386b5 +Subproject commit 649de7f382765dcff437ac7c0028c8da8df7276e From 994d8dae5a89cbb7a54f17c2d97ed6f056ec6c6a Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 9 Nov 2023 13:18:51 +0100 Subject: [PATCH 37/40] update run_multimer_jobs with the latest updates on alphalink inference --- alphapulldown/run_multimer_jobs.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/alphapulldown/run_multimer_jobs.py b/alphapulldown/run_multimer_jobs.py index afb33f70..62c7f6f8 100755 --- a/alphapulldown/run_multimer_jobs.py +++ b/alphapulldown/run_multimer_jobs.py @@ -346,21 +346,16 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ assert FLAGS.crosslinks is not None assert FLAGS.alphalink_weight is not None from unifold.alphalink_inference import alphalink_prediction - from unifold.dataset import process_ap + from unifold.config import model_config logging.info(f"Start using AlphaLink weights and cross-link information") MODEL_NAME = 'model_5_ptm_af2' configs = model_config(MODEL_NAME) - processed_features,_ = process_ap(config=configs.data, - features=multimer_object.feature_dict, - mode="predict",labels=None, - seed=42,batch_idx=None, - data_idx=None,is_distillation=False, - chain_id_map = multimer_object.chain_id_map, - crosslinks = FLAGS.crosslinks - ) - alphalink_prediction(processed_features,os.path.join(FLAGS.output_path,multimer_object.description), - param_path = FLAGS.alphalink_weight) + alphalink_prediction(multimer_object.feature_dict, + os.path.join(FLAGS.output_path,multimer_object.description), + param_path = FLAGS.alphalink_weight, + configs = configs,crosslinks=FLAGS.crosslinks, + chain_id_map=multimer_object.chain_id_map) else: predict( model_runners, From 9d5a66a3f72b4670991c9d1c5734474373a75dd8 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 9 Nov 2023 14:58:21 +0100 Subject: [PATCH 38/40] update unifold --- unifold | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unifold b/unifold index 649de7f3..fd900de4 160000 --- a/unifold +++ b/unifold @@ -1 +1 @@ -Subproject commit 649de7f382765dcff437ac7c0028c8da8df7276e +Subproject commit fd900de443bd76523f4e1ab1ef176f0b80dae0d7 From 728b9414757fd3591f591ba7d9fb92602ba728a6 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 9 Nov 2023 15:54:07 +0100 Subject: [PATCH 39/40] update pae ploting codes for alphalink inference --- alphapulldown/plot_pae.py | 31 ++++++++++++++++++++++++++++++ alphapulldown/run_multimer_jobs.py | 1 + 2 files changed, 32 insertions(+) diff --git a/alphapulldown/plot_pae.py b/alphapulldown/plot_pae.py index bf12f885..139bf785 100644 --- a/alphapulldown/plot_pae.py +++ b/alphapulldown/plot_pae.py @@ -57,3 +57,34 @@ def plot_pae(seqs: list, order, feature_dir, job_name): ax1.axvline(t, color="black", linewidth=3.5) plt.title("ranked_{}".format(i)) plt.savefig(f"{feature_dir}/{job_name}_PAE_plot_ranked_{i}.png") + +def plot_pae_from_matrix(seqs,pae_matrix,figure_name=''): + xticks = [] + initial_tick = 0 + for s in seqs: + initial_tick = initial_tick + len(s) + xticks.append(initial_tick) + + xticks_labels = [] + for i, t in enumerate(xticks): + xticks_labels.append(str(i + 1)) + + yticks_labels = [] + for s in seqs: + yticks_labels.append(str(len(s))) + fig, ax1 = plt.subplots(1, 1) + # plt.figure(figsize=(3,18)) + check = pae_matrix + fig, ax1 = plt.subplots(1, 1) + pos = ax1.imshow(check, cmap="bwr", vmin=0, vmax=30) + ax1.set_xticks(xticks) + ax1.set_yticks(xticks) + + ax1.set_xticklabels(xticks_labels, size="large") + ax1.set_yticklabels(yticks_labels,size="large") + fig.colorbar(pos).ax.set_title("unit: Angstrom") + for t in xticks: + ax1.axhline(t, color="black", linewidth=3.5) + ax1.axvline(t, color="black", linewidth=3.5) + plt.title("ranked_{}".format(i)) + plt.savefig(figure_name) \ No newline at end of file diff --git a/alphapulldown/run_multimer_jobs.py b/alphapulldown/run_multimer_jobs.py index 62c7f6f8..00bb3aef 100755 --- a/alphapulldown/run_multimer_jobs.py +++ b/alphapulldown/run_multimer_jobs.py @@ -353,6 +353,7 @@ def predict_individual_jobs(multimer_object, output_path, model_runners, random_ configs = model_config(MODEL_NAME) alphalink_prediction(multimer_object.feature_dict, os.path.join(FLAGS.output_path,multimer_object.description), + input_seqs = multimer_object.input_seqs, param_path = FLAGS.alphalink_weight, configs = configs,crosslinks=FLAGS.crosslinks, chain_id_map=multimer_object.chain_id_map) From 8434a59827dd079ba3574ecb57519293512f1663 Mon Sep 17 00:00:00 2001 From: Geoffrey Yu Date: Thu, 9 Nov 2023 16:06:45 +0100 Subject: [PATCH 40/40] update unifold --- unifold | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unifold b/unifold index fd900de4..7267885a 160000 --- a/unifold +++ b/unifold @@ -1 +1 @@ -Subproject commit fd900de443bd76523f4e1ab1ef176f0b80dae0d7 +Subproject commit 7267885a078d5878e4d4dfc25d42ef10af678354