I followed the readme instructions to get the miniconda environment and got the preprocessed data.
After running into the environment:
Validation sanity check: 100%|โโโโโโโโโโ| 1/1.0 [00:00<00:00, 1.88it/s]2020-11-18 09:23:26.893 | ERROR | __main__:<module>:299 - An error has been caught in function '<module>', process 'MainProcess' (16505), thread 'MainThread' (140044919113536):
Traceback (most recent call last):
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
โ โ โ {'__name__': '__main__', '__doc__': None, '__package__': 'src.graphqa', '__loader__': <_frozen_importlib_external.SourceFileL...
โ โ <code object <module> at 0x7f5ebf05db30, file "/storage/graphqa/src/graphqa/train.py", line 1>
โ <function _run_code at 0x7f5ebf03bee0>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
โ โ {'__name__': '__main__', '__doc__': None, '__package__': 'src.graphqa', '__loader__': <_frozen_importlib_external.SourceFileL...
โ <code object <module> at 0x7f5ebf05db30, file "/storage/graphqa/src/graphqa/train.py", line 1>
> File "/storage/graphqa/src/graphqa/train.py", line 299, in <module>
main()
โ <function main at 0x7f5e25a05c10>
File "/storage/graphqa/src/graphqa/train.py", line 292, in main
result = trainer.fit(model)
โ โ โ LightningGraphQA(
โ โ (model): RecursiveScriptModule(
โ โ original_name=GraphQA
โ โ (embeddings): RecursiveScriptModule(
โ โ ...
โ โ <function Trainer.fit at 0x7f5e5c5ff280>
โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f5e177f1790>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/trainer/states.py", line 48, in wrapped_fn
result = fn(self, *args, **kwargs)
โ โ โ โ {}
โ โ โ (LightningGraphQA(
โ โ (model): RecursiveScriptModule(
โ โ original_name=GraphQA
โ โ (embeddings): RecursiveScriptModule(
โ โ ...
โ โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f5e177f1790>
โ <function Trainer.fit at 0x7f5e5c5ff1f0>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1073, in fit
results = self.accelerator_backend.train(model)
โ โ โ โ LightningGraphQA(
โ โ โ (model): RecursiveScriptModule(
โ โ โ original_name=GraphQA
โ โ โ (embeddings): RecursiveScriptModule(
โ โ โ ...
โ โ โ <function GPUBackend.train at 0x7f5e6123be50>
โ โ <pytorch_lightning.accelerators.gpu_backend.GPUBackend object at 0x7f5e25922940>
โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f5e177f1790>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/accelerators/gpu_backend.py", line 51, in train
results = self.trainer.run_pretrain_routine(model)
โ โ โ โ LightningGraphQA(
โ โ โ (model): RecursiveScriptModule(
โ โ โ original_name=GraphQA
โ โ โ (embeddings): RecursiveScriptModule(
โ โ โ ...
โ โ โ <function Trainer.run_pretrain_routine at 0x7f5e5c5ff430>
โ โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f5e177f1790>
โ <pytorch_lightning.accelerators.gpu_backend.GPUBackend object at 0x7f5e25922940>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1224, in run_pretrain_routine
self._run_sanity_check(ref_model, model)
โ โ โ โ LightningGraphQA(
โ โ โ (model): RecursiveScriptModule(
โ โ โ original_name=GraphQA
โ โ โ (embeddings): RecursiveScriptModule(
โ โ โ ...
โ โ โ LightningGraphQA(
โ โ (model): RecursiveScriptModule(
โ โ original_name=GraphQA
โ โ (embeddings): RecursiveScriptModule(
โ โ ...
โ โ <function Trainer._run_sanity_check at 0x7f5e5c5ff4c0>
โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f5e177f1790>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1257, in _run_sanity_check
eval_results = self._evaluate(model, self.val_dataloaders, max_batches, False)
โ โ โ โ โ โ [1.0]
โ โ โ โ โ [<torch_geometric.data.dataloader.DataLoader object at 0x7f5e176ee2b0>]
โ โ โ โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f5e177f1790>
โ โ โ LightningGraphQA(
โ โ (model): RecursiveScriptModule(
โ โ original_name=GraphQA
โ โ (embeddings): RecursiveScriptModule(
โ โ ...
โ โ <function TrainerEvaluationLoopMixin._evaluate at 0x7f5e5c649310>
โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f5e177f1790>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 413, in _evaluate
self.__call_eval_loop_hook_end(test_mode)
โ โ False
โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f5e177f1790>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 237, in __call_eval_loop_hook_end
self.__call_eval_loop_hook_evt(test_mode, 'end')
โ โ False
โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f5e177f1790>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 247, in __call_eval_loop_hook_evt
getattr(self, hook_name)()
โ โ 'on_validation_epoch_end'
โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f5e177f1790>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/trainer/callback_hook.py", line 86, in on_validation_epoch_end
callback.on_validation_epoch_end(self, self.get_model())
โ โ โ โ โ <function TrainerIOMixin.get_model at 0x7f5e5c5d8f70>
โ โ โ โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f5e177f1790>
โ โ โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f5e177f1790>
โ โ <function EarlyStopping.on_validation_epoch_end at 0x7f5e6128b5e0>
โ <src.graphqa.callbacks.MyEarlyStopping object at 0x7f5e177f1910>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/callbacks/early_stopping.py", line 156, in on_validation_epoch_end
self._validate_condition_metric(trainer.callback_metrics)
โ โ โ โ {'global': score cad gdtha gdtts lddt tm
โ โ โ metric ...
โ โ โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f5e177f1790>
โ โ <function EarlyStopping._validate_condition_metric at 0x7f5e6128b310>
โ <src.graphqa.callbacks.MyEarlyStopping object at 0x7f5e177f1910>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/callbacks/early_stopping.py", line 117, in _validate_condition_metric
raise RuntimeError(error_msg)
โ 'Early stopping conditioned on metric `val/global_per_target/gdtts/R` which is not available. Either add `val/global_per_targ...
RuntimeError: Early stopping conditioned on metric `val/global_per_target/gdtts/R` which is not available. Either add `val/global_per_target/gdtts/R` to the return of validation_epoch end or modify your EarlyStopping callback to use any of the following: `global`, `local`, `global_per_target`, `local_per_decoy`, `global/gdtts`, `global/cad`, `local/lddt`, `local/cad`
So I tried to substitute "val/global_per_target/gdtts/R" with "global/gdtts" in config/train.yaml but then I get:
Epoch 0: 0%| | 1/694 [00:05<1:00:18, 5.22s/it, loss=0.550]2020-11-18 09:34:34.125 | ERROR | __main__:<module>:299 - An error has been caught in function '<module>', process 'MainProcess' (16806), thread 'MainThread' (140263578617664):
Traceback (most recent call last):
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
โ โ โ {'__name__': '__main__', '__doc__': None, '__package__': 'src.graphqa', '__loader__': <_frozen_importlib_external.SourceFileL...
โ โ <code object <module> at 0x7f91a8251b30, file "/storage/graphqa/src/graphqa/train.py", line 1>
โ <function _run_code at 0x7f91a822fee0>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
โ โ {'__name__': '__main__', '__doc__': None, '__package__': 'src.graphqa', '__loader__': <_frozen_importlib_external.SourceFileL...
โ <code object <module> at 0x7f91a8251b30, file "/storage/graphqa/src/graphqa/train.py", line 1>
> File "/storage/graphqa/src/graphqa/train.py", line 299, in <module>
main()
โ <function main at 0x7f910ebf9c10>
File "/storage/graphqa/src/graphqa/train.py", line 292, in main
result = trainer.fit(model)
โ โ โ LightningGraphQA(
โ โ (model): RecursiveScriptModule(
โ โ original_name=GraphQA
โ โ (embeddings): RecursiveScriptModule(
โ โ ...
โ โ <function Trainer.fit at 0x7f91457f3280>
โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f910ca17790>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/trainer/states.py", line 48, in wrapped_fn
result = fn(self, *args, **kwargs)
โ โ โ โ {}
โ โ โ (LightningGraphQA(
โ โ (model): RecursiveScriptModule(
โ โ original_name=GraphQA
โ โ (embeddings): RecursiveScriptModule(
โ โ ...
โ โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f910ca17790>
โ <function Trainer.fit at 0x7f91457f31f0>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1073, in fit
results = self.accelerator_backend.train(model)
โ โ โ โ LightningGraphQA(
โ โ โ (model): RecursiveScriptModule(
โ โ โ original_name=GraphQA
โ โ โ (embeddings): RecursiveScriptModule(
โ โ โ ...
โ โ โ <function GPUBackend.train at 0x7f914a42ee50>
โ โ <pytorch_lightning.accelerators.gpu_backend.GPUBackend object at 0x7f910eb19940>
โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f910ca17790>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/accelerators/gpu_backend.py", line 51, in train
results = self.trainer.run_pretrain_routine(model)
โ โ โ โ LightningGraphQA(
โ โ โ (model): RecursiveScriptModule(
โ โ โ original_name=GraphQA
โ โ โ (embeddings): RecursiveScriptModule(
โ โ โ ...
โ โ โ <function Trainer.run_pretrain_routine at 0x7f91457f3430>
โ โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f910ca17790>
โ <pytorch_lightning.accelerators.gpu_backend.GPUBackend object at 0x7f910eb19940>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1239, in run_pretrain_routine
self.train()
โ โ <function TrainerTrainLoopMixin.train at 0x7f91457df310>
โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f910ca17790>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 394, in train
self.run_training_epoch()
โ โ <function TrainerTrainLoopMixin.run_training_epoch at 0x7f91457df4c0>
โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f910ca17790>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 529, in run_training_epoch
monitor_metrics = deepcopy(self.callback_metrics)
โ โ โ {'val_scores': {'global': score tm ... cad
โ โ ...
โ โ <pytorch_lightning.trainer.trainer.Trainer object at 0x7f910ca17790>
โ <function deepcopy at 0x7f914bc543a0>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/copy.py", line 146, in deepcopy
y = copier(x, memo)
โ โ โ <unprintable dict object>
โ โ {'val_scores': {'global': score tm ... cad
โ ...
โ <function _deepcopy_dict at 0x7f914bc545e0>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/copy.py", line 230, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
โ โ โ โ โ โ โ <unprintable dict object>
โ โ โ โ โ โ {'global/gdtts': <Figure size 300x300 with 1 Axes>, 'global/cad': <Figure size 300x300 with 1 Axes>, 'local/lddt': <Figure si...
โ โ โ โ โ <function deepcopy at 0x7f914bc543a0>
โ โ โ โ <unprintable dict object>
โ โ โ 'val_figures'
โ โ <function deepcopy at 0x7f914bc543a0>
โ {'val_scores': {'global': score tm ... cad
...
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/copy.py", line 146, in deepcopy
y = copier(x, memo)
โ โ โ <unprintable dict object>
โ โ {'global/gdtts': <Figure size 300x300 with 1 Axes>, 'global/cad': <Figure size 300x300 with 1 Axes>, 'local/lddt': <Figure si...
โ <function _deepcopy_dict at 0x7f914bc545e0>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/copy.py", line 230, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
โ โ โ โ โ โ โ <unprintable dict object>
โ โ โ โ โ โ <Figure size 300x300 with 1 Axes>
โ โ โ โ โ <function deepcopy at 0x7f914bc543a0>
โ โ โ โ <unprintable dict object>
โ โ โ 'global/gdtts'
โ โ <function deepcopy at 0x7f914bc543a0>
โ {}
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/copy.py", line 172, in deepcopy
y = _reconstruct(x, memo, *rv)
โ โ โ โ (<function __newobj__ at 0x7f91a81ac790>, (<class 'matplotlib.figure.Figure'>,), {'_stale': True, 'stale_callback': None, 'fi...
โ โ โ <unprintable dict object>
โ โ <Figure size 300x300 with 1 Axes>
โ <function _reconstruct at 0x7f914bc54790>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/copy.py", line 270, in _reconstruct
state = deepcopy(state, memo)
โ โ โ <unprintable dict object>
โ โ {'_stale': True, 'stale_callback': None, 'figure': None, '_transform': None, '_transformSet': False, '_visible': True, '_anim...
โ <function deepcopy at 0x7f914bc543a0>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/copy.py", line 146, in deepcopy
y = copier(x, memo)
โ โ โ <unprintable dict object>
โ โ {'_stale': True, 'stale_callback': None, 'figure': None, '_transform': None, '_transformSet': False, '_visible': True, '_anim...
โ <function _deepcopy_dict at 0x7f914bc545e0>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/copy.py", line 230, in _deepcopy_dict
y[deepcopy(key, memo)] = deepcopy(value, memo)
โ โ โ โ โ โ โ <unprintable dict object>
โ โ โ โ โ โ Bbox([[0.0, 0.0], [3.0, 3.0]])
โ โ โ โ โ <function deepcopy at 0x7f914bc543a0>
โ โ โ โ <unprintable dict object>
โ โ โ 'bbox_inches'
โ โ <function deepcopy at 0x7f914bc543a0>
โ {'_stale': True, 'stale_callback': None, 'figure': None, '_transform': None, '_transformSet': False, '_visible': True, '_anim...
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/copy.py", line 153, in deepcopy
y = copier(memo)
โ โ <unprintable dict object>
โ <bound method TransformNode.__copy__ of Bbox([[0.0, 0.0], [3.0, 3.0]])>
File "/home/gabriele/miniconda3/envs/graphqa/lib/python3.8/site-packages/matplotlib/transforms.py", line 143, in __copy__
raise NotImplementedError(
NotImplementedError: TransformNode instances can not be copied. Consider using frozen() instead.