# Copyright 2022-2025 The Ramble Authors
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
import os
import pytest
import ramble.context
import ramble.experiment_set
import ramble.renderer
import ramble.workspace
from ramble.application import ChainCycleDetectedError, InvalidChainError
from ramble.main import RambleCommand
pytestmark = pytest.mark.usefixtures(
"mutable_config",
"mutable_mock_workspace_path",
"mutable_mock_apps_repo",
)
workspace = RambleCommand("workspace")
[docs]
def test_single_experiment_in_set(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {
"exp_var1": "1",
"exp_var2": "2",
"n_nodes": "2",
}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
[docs]
def test_vector_experiment_in_set(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "4"]}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8" in exp_set.experiments.keys()
[docs]
def test_vector_length_mismatch_errors(request, mutable_mock_workspace_path, capsys):
ws_name = request.node.name
workspace("create", ws_name)
assert ws_name in workspace("list")
with ramble.workspace.read(ws_name) as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"processes_per_node": ["2", "4", "6"]}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}_{processes_per_node}"
experiment_context.variables = {"n_nodes": ["2", "4"]}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
with pytest.raises(SystemExit):
exp_set.set_experiment_context(experiment_context)
captured = capsys.readouterr().err
assert "Length mismatch in vector variables in experiment series1_{n_ranks}" in captured
assert "Variable processes_per_node has length 3" in captured
assert "Variable n_nodes has length 2" in captured
[docs]
def test_nonunique_vector_errors(mutable_mock_workspace_path, capsys):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{processes_per_node}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "4"]}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
with pytest.raises(SystemExit):
exp_set.set_experiment_context(experiment_context)
captured = capsys.readouterr().err
assert "is not unique." in captured
[docs]
def test_zipped_vector_experiments(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
"processes_per_node": ["2", "4"],
}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}_{processes_per_node}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "4"]}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4_2" in exp_set.experiments.keys()
assert "basic.test_wl.series1_16_4" in exp_set.experiments.keys()
[docs]
def test_matrix_experiments(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "3"]}
experiment_context.matrices = [["n_nodes"]]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
assert "basic.test_wl.series1_6" in exp_set.experiments.keys()
[docs]
def test_matrix_multiplication_experiments(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
"processes_per_node": ["1", "4", "6"],
}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "4"]}
experiment_context.matrices = [["n_nodes", "processes_per_node"]]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_2" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8" in exp_set.experiments.keys()
assert "basic.test_wl.series1_12" in exp_set.experiments.keys()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
assert "basic.test_wl.series1_16" in exp_set.experiments.keys()
assert "basic.test_wl.series1_24" in exp_set.experiments.keys()
[docs]
def test_matrix_vector_experiments(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
"processes_per_node": ["2", "4"],
}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "3"]}
experiment_context.matrices = [["n_nodes"]]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8" in exp_set.experiments.keys()
assert "basic.test_wl.series1_6" in exp_set.experiments.keys()
assert "basic.test_wl.series1_12" in exp_set.experiments.keys()
[docs]
def test_multi_matrix_experiments(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
"processes_per_node": ["2", "4"],
}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}_{processes_per_node}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "3"]}
experiment_context.matrices = [["n_nodes"], ["processes_per_node"]]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4_2" in exp_set.experiments.keys()
assert "basic.test_wl.series1_12_4" in exp_set.experiments.keys()
[docs]
def test_full_experiments_from_dict(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
app_dict = {
"variables": {
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
}
wl_dict = {"variables": {"partition": ["p1", "p2"], "processes_per_node": ["2", "4"]}}
exp_dict = {
"variables": {"n_nodes": ["2", "3"], "ids": ["1", "2", "3", "4"]},
"zips": {"part_conf": ["partition", "processes_per_node"]},
"matrices": [{"mat_name": ["part_conf", "n_nodes"]}],
"matrix": ["ids"],
}
appContext = ramble.context.create_context_from_dict("basic", app_dict)
wlContext = ramble.context.create_context_from_dict("test_wl", wl_dict)
expContext = ramble.context.create_context_from_dict(
"test_{n_ranks}_{processes_per_node}_{ids}", exp_dict
)
exp_set.set_application_context(appContext)
exp_set.set_workload_context(wlContext)
exp_set.set_experiment_context(expContext)
assert "basic.test_wl.test_4_2_1" in exp_set.experiments.keys()
assert "basic.test_wl.test_6_2_2" in exp_set.experiments.keys()
assert "basic.test_wl.test_8_4_3" in exp_set.experiments.keys()
assert "basic.test_wl.test_12_4_4" in exp_set.experiments.keys()
assert exp_set._context[exp_set._contexts.workspace].context_name is not None
[docs]
def test_matrix_undefined_var_errors(mutable_mock_workspace_path, capsys):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
"processes_per_node": ["2", "4"],
}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}_{processes_per_node}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "3"]}
experiment_context.matrices = [["n_nodes"], ["foo"]]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
with pytest.raises(SystemExit):
exp_set.set_experiment_context(experiment_context)
captured = capsys.readouterr().err
assert "variable or zip foo has not been defined yet." in captured
[docs]
def test_experiment_names_match(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
"processes_per_node": ["2", "4"],
}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}_{processes_per_node}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "3"]}
experiment_context.matrices = [["n_nodes"], ["processes_per_node"]]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4_2" in exp_set.experiments.keys()
assert "basic.test_wl.series1_12_4" in exp_set.experiments.keys()
for exp, app, _ in exp_set.all_experiments():
assert exp == app.expander.expand_var("{experiment_namespace}")
[docs]
def test_cross_experiment_variable_references(request, mutable_mock_workspace_path):
ws_name = request.node.name
workspace("create", ws_name)
assert ws_name in workspace("list")
with ramble.workspace.read(ws_name) as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment1_context = ramble.context.Context()
experiment1_context.context_name = "series1_{n_ranks}"
experiment1_context.variables = {
"exp_var1": "1",
"exp_var2": "2",
"n_nodes": "2",
"test_var": "success",
}
experiment1_context.internals = {
"custom_executables": {
"echo_test": {"template": 'echo "{test_var}"'},
},
"executable_injection": [{"name": "echo_test", "order": "after"}],
}
experiment2_context = ramble.context.Context()
experiment2_context.context_name = "series2_{n_ranks}"
experiment2_context.variables = {
"exp_var1": "1",
"exp_var2": "2",
"n_nodes": "2",
"test_var": "test_var in basic.test_wl.series1_4",
}
experiment2_context.internals = {
"custom_executables": {
"echo_test": {"template": 'echo "{test_var}"'},
},
"executable_injection": [{"name": "echo_test", "order": "after"}],
}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment1_context)
exp_set.set_experiment_context(experiment2_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
assert "basic.test_wl.series2_4" in exp_set.experiments.keys()
exp2_app = exp_set.experiments["basic.test_wl.series2_4"]
assert exp2_app.expander.expand_var("{test_var}") == "success"
[docs]
def test_cross_experiment_missing_experiment_errors(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment1_context = ramble.context.Context()
experiment1_context.context_name = "series1_{n_ranks}"
experiment1_context.variables = {
"exp_var1": "1",
"exp_var2": "2",
"n_nodes": "2",
"test_var": "processes_per_node in basic.test_wl.does_not_exist",
}
experiment1_context.internals = {
"custom_executables": {
"echo_test": {"template": 'echo "{test_var}"'},
},
"executable_injection": [{"name": "echo_test", "order": "after"}],
}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
expected = (
"basic.test_wl.does_not_exist does not exist in: "
f'"{experiment1_context.variables["test_var"]}"'
)
with pytest.raises(ramble.expander.RambleSyntaxError, match=expected):
exp_set.set_experiment_context(experiment1_context)
[docs]
def test_n_ranks_correct_defaults(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "3"]}
experiment_context.matrices = [["n_nodes"]]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
assert "basic.test_wl.series1_6" in exp_set.experiments.keys()
[docs]
def test_n_nodes_correct_defaults(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": ["4", "6"],
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}_{n_nodes}"
experiment_context.variables = {
"exp_var1": "1",
"exp_var2": "2",
}
experiment_context.matrices = [["n_ranks"]]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4_2" in exp_set.experiments.keys()
assert "basic.test_wl.series1_6_3" in exp_set.experiments.keys()
[docs]
def test_processes_per_node_correct_defaults(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
# Remove workspace vars, which default to a `processes_per_node = -1` definition.
exp_set._context[exp_set._contexts.workspace].variables = {}
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": ["4", "6"],
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}_{processes_per_node}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "3"]}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4_2" in exp_set.experiments.keys()
assert "basic.test_wl.series1_6_2" in exp_set.experiments.keys()
[docs]
@pytest.mark.parametrize("var", ["env_path"])
def test_reserved_keywords_error_in_application(mutable_mock_workspace_path, var):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": ["4", "6"],
var: "should_fail",
"mpi_command": "",
"batch_submit": "",
}
with pytest.raises(ramble.experiment_set.RambleVariableDefinitionError) as e:
exp_set.set_application_context(application_context)
captured = str(e.value)
assert "In application basic" in captured
assert f"{var}" in captured
assert "is reserved by ramble" in captured
[docs]
@pytest.mark.parametrize("var", ["env_path"])
def test_reserved_keywords_error_in_workload(mutable_mock_workspace_path, var):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": ["4", "6"],
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", var: "should_fail"}
exp_set.set_application_context(application_context)
with pytest.raises(ramble.experiment_set.RambleVariableDefinitionError) as e:
exp_set.set_workload_context(workload_context)
captured = str(e.value)
assert "In workload basic.test_wl" in captured
assert f"{var}" in captured
assert "is reserved by ramble" in captured
[docs]
@pytest.mark.parametrize("var", ["env_path"])
def test_reserved_keywords_error_in_experiment(mutable_mock_workspace_path, var):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
# Remove workspace vars, which default to a `processes_per_node = -1` definition.
exp_set._context[exp_set._contexts.base].variables = {}
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": ["4", "6"],
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}_{processes_per_node}"
experiment_context.variables = {
"exp_var1": "1",
"exp_var2": "2",
"n_nodes": ["2", "3"],
var: "should_fail",
}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
with pytest.raises(ramble.experiment_set.RambleVariableDefinitionError) as e:
exp_set.set_experiment_context(experiment_context)
captured = str(e.value)
assert "In experiment basic.test_wl" in captured
assert f"{var}" in captured
assert "is reserved by ramble" in captured
[docs]
@pytest.mark.parametrize("var", ["batch_submit", "mpi_command"])
def test_missing_required_keyword_errors(mutable_mock_workspace_path, var, capsys):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
for context in exp_set._contexts:
if exp_set._context[context].variables and var in exp_set._context[context].variables:
del exp_set._context[context].variables[var]
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": ["4", "6"],
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}_{processes_per_node}"
experiment_context.variables = {
"exp_var1": "1",
"exp_var2": "2",
"n_nodes": ["2", "3"],
"batch_submit": "",
"mpi_command": "",
}
if var in experiment_context.variables.keys():
del experiment_context.variables[var]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
with pytest.raises(ramble.experiment_set.RambleVariableDefinitionError):
exp_set.set_experiment_context(experiment_context)
captured = capsys.readouterr()
assert f'Required key "{var}" is not defined' in captured.err
assert "in basic.test_wl.series1_{n_ranks}_{processes_per_node}" in captured.err
[docs]
def test_chained_experiments_populate_new_experiments(mutable_mock_workspace_path, capsys):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"processes_per_node": "1",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
}
experiment1_context = ramble.context.Context()
experiment1_context.context_name = "test1"
experiment1_context.variables = {"n_ranks": "2"}
experiment2_context = ramble.context.Context()
experiment2_context.context_name = "series2_{n_ranks}"
experiment2_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_ranks": ["4", "6"]}
experiment2_context.chained_experiments = [
{
"name": "basic.test_wl.test1",
"order": "before_root",
"command": "{execute_experiment}",
"variables": {},
},
{
"name": "basic.test_wl.test1",
"order": "after_root",
"command": "{execute_experiment}",
"variables": {},
},
]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment1_context)
exp_set.set_experiment_context(experiment2_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series2_4" in exp_set.experiments
assert "basic.test_wl.series2_4.chain.0.basic.test_wl.test1" in exp_set.chained_experiments
assert "basic.test_wl.series2_4.chain.1.basic.test_wl.test1" in exp_set.chained_experiments
assert "basic.test_wl.series2_6" in exp_set.experiments
assert "basic.test_wl.series2_6.chain.0.basic.test_wl.test1" in exp_set.chained_experiments
assert "basic.test_wl.series2_6.chain.1.basic.test_wl.test1" in exp_set.chained_experiments
assert "basic.test_wl.test1" in exp_set.experiments
[docs]
def test_chained_experiment_has_correct_directory(mutable_mock_workspace_path, capsys):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"processes_per_node": "1",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
}
experiment1_context = ramble.context.Context()
experiment1_context.context_name = "test1"
experiment1_context.variables = {"n_ranks": "2"}
experiment2_context = ramble.context.Context()
experiment2_context.context_name = "series2_{n_ranks}"
experiment2_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_ranks": "4"}
experiment2_context.chained_experiments = [
{
"name": "basic.test_wl.test1",
"order": "before_root",
"command": "{execute_experiment}",
"variables": {},
},
]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment1_context)
exp_set.set_experiment_context(experiment2_context)
exp_set.build_experiment_chains()
parent_name = "basic.test_wl.series2_4"
chained_name = "basic.test_wl.series2_4.chain.0.basic.test_wl.test1"
chained_dir = "chain.0.basic.test_wl.test1"
assert parent_name in exp_set.experiments
assert chained_name in exp_set.chained_experiments
parent_inst = exp_set.get_experiment(parent_name)
chained_inst = exp_set.get_experiment(chained_name)
parent_run_dir = parent_inst.expander.expand_var("{experiment_run_dir}")
expected_dir = os.path.join(parent_run_dir, "chained_experiments", chained_dir)
assert chained_inst.variables["experiment_run_dir"] == expected_dir
[docs]
def test_chained_cycle_errors(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"processes_per_node": "1",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
}
experiment1_context = ramble.context.Context()
experiment1_context.context_name = "test1"
experiment1_context.variables = {"n_ranks": "2"}
experiment2_context = ramble.context.Context()
experiment2_context.context_name = "series2_{n_ranks}"
experiment2_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_ranks": "4"}
experiment2_context.chained_experiments = [
{
"name": "basic.test_wl.series2_4",
"order": "before_root",
"command": "{execute_experiment}",
"variables": {},
},
]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment1_context)
exp_set.set_experiment_context(experiment2_context)
with pytest.raises(ChainCycleDetectedError, match="Cycle detected in experiment chain"):
exp_set.build_experiment_chains()
[docs]
def test_chained_invalid_order_errors(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"processes_per_node": "1",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
}
experiment1_context = ramble.context.Context()
experiment1_context.context_name = "test1"
experiment1_context.variables = {"n_ranks": "2"}
experiment2_context = ramble.context.Context()
experiment2_context.context_name = "series2_{n_ranks}"
experiment2_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_ranks": "4"}
experiment2_context.chained_experiments = [
{
"name": "basic.test_wl.test1",
"order": "foo",
"command": "{execute_experiment}",
"variables": {},
},
]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment1_context)
exp_set.set_experiment_context(experiment2_context)
with pytest.raises(InvalidChainError, match="Invalid experiment chain defined:"):
exp_set.build_experiment_chains()
[docs]
def test_modifiers_set_correctly(mutable_mock_workspace_path, mock_modifiers, capsys):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"processes_per_node": "1",
"mpi_command": "",
"batch_submit": "",
}
application_context.modifiers = [
{"name": "test-mod", "mode": "app-scope", "on_executable": ["builtin::env_vars"]}
]
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
}
workload_context.modifiers = [
{"name": "test-mod", "mode": "wl-scope", "on_executable": ["builtin::env_vars"]}
]
experiment_context = ramble.context.Context()
experiment_context.context_name = "test1"
experiment_context.variables = {"n_ranks": "2"}
experiment_context.modifiers = [
{"name": "test-mod", "mode": "exp-scope", "on_executable": ["builtin::env_vars"]}
]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
assert "basic.test_wl.test1" in exp_set.experiments
app_inst = exp_set.experiments["basic.test_wl.test1"]
assert app_inst.modifiers is not None
expected_modifier_modes = {"app-scope", "wl-scope", "exp-scope"}
for mod_def in app_inst.modifiers:
assert mod_def["mode"] in expected_modifier_modes
expected_modifier_modes.remove(mod_def["mode"])
assert len(expected_modifier_modes) == 0
[docs]
def test_explicit_zips_work(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "4"]}
experiment_context.zips = {"test_zip": ["n_nodes"]}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8" in exp_set.experiments.keys()
[docs]
def test_explicit_zips_in_matrix(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}_{exp_var1}"
experiment_context.variables = {
"exp_var1": ["1", "a", "3"],
"exp_var2": ["2", "b", "4"],
"n_nodes": ["2", "4"],
}
experiment_context.matrices = [["test_zip"]]
experiment_context.zips = {"test_zip": ["exp_var1", "exp_var2"]}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4_1" in exp_set.experiments.keys()
assert "basic.test_wl.series1_4_a" in exp_set.experiments.keys()
assert "basic.test_wl.series1_4_3" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8_1" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8_a" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8_3" in exp_set.experiments.keys()
[docs]
def test_explicit_zips_unconsumed(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}_{exp_var1}"
experiment_context.variables = {
"exp_var1": ["1", "a", "3"],
"exp_var2": ["2", "b", "4"],
"n_nodes": ["2", "4"],
}
experiment_context.matrices = [["n_nodes"]]
experiment_context.zips = {"test_zip": ["exp_var1", "exp_var2"]}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4_1" in exp_set.experiments.keys()
assert "basic.test_wl.series1_4_a" in exp_set.experiments.keys()
assert "basic.test_wl.series1_4_3" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8_1" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8_a" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8_3" in exp_set.experiments.keys()
[docs]
def test_single_var_explicit_zip(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "4"]}
experiment_context.zips = {
"test_zip": ["n_nodes"],
}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8" in exp_set.experiments.keys()
[docs]
def test_zip_multi_use_var_errors(mutable_mock_workspace_path, capsys):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "4"]}
experiment_context.zips = {
"test_zip1": ["n_nodes"],
"test_zip2": ["n_nodes"],
}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
with pytest.raises(SystemExit):
exp_set.set_experiment_context(experiment_context)
captured = capsys.readouterr().err
assert "An undefined variable n_nodes is defined in zip test_zip2" in captured
[docs]
def test_zip_non_list_var_errors(mutable_mock_workspace_path, capsys):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {"n_nodes": ["2", "4"], "exp_var1": ["1", "2"]}
experiment_context.zips = {
"test_zip": ["exp_var1", "processes_per_node"],
}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
with pytest.raises(SystemExit):
exp_set.set_experiment_context(experiment_context)
captured = capsys.readouterr().err
assert "Variable processes_per_node in zip test_zip does not refer to a vector" in captured
[docs]
def test_zip_variable_lengths_errors(mutable_mock_workspace_path, capsys):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {
"exp_var1": "1",
"processes_per_node": ["2"],
"n_nodes": ["2", "4"],
}
experiment_context.zips = {
"test_zip": ["n_nodes", "processes_per_node"],
}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
with pytest.raises(SystemExit):
exp_set.set_experiment_context(experiment_context)
captured = capsys.readouterr().err
assert "Variable processes_per_node in zip" in captured
assert "has a length of 1" in captured
assert "differs from the current max of 2" in captured
[docs]
def test_vector_experiment_with_explicit_excludes(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "4"]}
experiment_context.exclude = {"variables": {"n_nodes": ["4"]}}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8" not in exp_set.experiments.keys()
[docs]
def test_matrix_experiments_explicit_excludes(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {"exp_var1": "1", "exp_var2": "2", "n_nodes": ["2", "3"]}
experiment_context.matrices = [["n_nodes"]]
experiment_context.exclude = {
"variables": {
"n_nodes": ["3"],
},
"matrix": ["n_nodes"],
}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
assert "basic.test_wl.series1_6" not in exp_set.experiments.keys()
[docs]
def test_vector_experiment_with_where_excludes(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {
"exp_var1": "1",
"exp_var2": "2",
"n_nodes": ["1", "2", "3", "4", "5"],
}
experiment_context.exclude = {"where": ["{n_nodes} > 2 and {n_nodes} < 5"]}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_2" in exp_set.experiments.keys()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
assert "basic.test_wl.series1_6" not in exp_set.experiments.keys()
assert "basic.test_wl.series1_8" not in exp_set.experiments.keys()
assert "basic.test_wl.series1_10" in exp_set.experiments.keys()
[docs]
def test_vector_experiment_with_multi_where_excludes(mutable_mock_workspace_path):
workspace("create", "test")
assert "test" in workspace("list")
with ramble.workspace.read("test") as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {
"exp_var1": "1",
"exp_var2": "2",
"n_nodes": ["1", "2", "3", "4", "5"],
}
experiment_context.exclude = {"where": ["{n_nodes} < 2", "{n_nodes} > 4"]}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_2" not in exp_set.experiments.keys()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
assert "basic.test_wl.series1_6" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8" in exp_set.experiments.keys()
assert "basic.test_wl.series1_10" not in exp_set.experiments.keys()
[docs]
def test_unused_vector_no_error(request, mutable_mock_workspace_path):
ws_name = request.node.name
workspace("create", ws_name)
assert ws_name in workspace("list")
with ramble.workspace.read(ws_name) as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {
"exp_var1": "1",
"exp_var2": "2",
"other_var": ["1", "2,", "3"],
"n_nodes": ["1", "2", "3", "4", "5"],
}
experiment_context.exclude = {"where": ["{n_nodes} < 2", "{n_nodes} > 4"]}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_2" not in exp_set.experiments.keys()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
assert "basic.test_wl.series1_6" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8" in exp_set.experiments.keys()
assert "basic.test_wl.series1_10" not in exp_set.experiments.keys()
[docs]
def test_unused_zip_no_error(request, mutable_mock_workspace_path):
ws_name = request.node.name
workspace("create", ws_name)
assert ws_name in workspace("list")
with ramble.workspace.read(ws_name) as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"n_ranks": "{processes_per_node}*{n_nodes}",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {"wl_var1": "1", "wl_var2": "2", "processes_per_node": "2"}
experiment_context = ramble.context.Context()
experiment_context.context_name = "series1_{n_ranks}"
experiment_context.variables = {
"exp_var1": "1",
"exp_var2": "2",
"other_var_1": ["1", "2,", "3"],
"other_var_2": ["1", "2,", "3"],
"n_nodes": ["1", "2", "3", "4", "5"],
}
experiment_context.zips = {
"test_zip": ["other_var_1", "other_var_2"],
}
experiment_context.exclude = {"where": ["{n_nodes} < 2", "{n_nodes} > 4"]}
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.series1_2" not in exp_set.experiments.keys()
assert "basic.test_wl.series1_4" in exp_set.experiments.keys()
assert "basic.test_wl.series1_6" in exp_set.experiments.keys()
assert "basic.test_wl.series1_8" in exp_set.experiments.keys()
assert "basic.test_wl.series1_10" not in exp_set.experiments.keys()
[docs]
def test_unused_var_propagates_to_chain(request, mutable_mock_workspace_path):
ws_name = request.node.name
workspace("create", ws_name)
assert ws_name in workspace("list")
with ramble.workspace.read(ws_name) as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
expanded_app_context = ramble.context.Context()
expanded_app_context.context_name = "expanded_foms"
expanded_app_context.variables = {
"processes_per_node": "1",
"mpi_command": "",
"batch_submit": "",
}
expanded_wl_context = ramble.context.Context()
expanded_wl_context.context_name = "test_wl"
expanded_wl_context.variables = {}
expanded_exp_context = ramble.context.Context()
expanded_exp_context.context_name = "test"
expanded_exp_context.variables = {"n_ranks": "1", "my_var": "2.0"}
expanded_exp_context.is_template = True
exp_set.set_application_context(expanded_app_context)
exp_set.set_workload_context(expanded_wl_context)
exp_set.set_experiment_context(expanded_exp_context)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"processes_per_node": "1",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
}
experiment_context = ramble.context.Context()
experiment_context.context_name = "test1"
experiment_context.variables = {"n_ranks": "2", "my_var": "5.0"}
experiment_context.chained_experiments = [
{
"name": "expanded_foms.test_wl.test",
"order": "after_root",
"command": "{execute_experiment}",
"inherit_variables": ["my_var"],
},
]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.test1" in exp_set.experiments
assert (
"basic.test_wl.test1.chain.0.expanded_foms.test_wl.test" in exp_set.chained_experiments
)
app_inst = exp_set.get_experiment("basic.test_wl.test1.chain.0.expanded_foms.test_wl.test")
assert app_inst.variables["my_var"] == "5.0"
[docs]
def test_custom_executables_track_variables(request, mutable_mock_workspace_path):
ws_name = request.node.name
workspace("create", ws_name)
assert ws_name in workspace("list")
with ramble.workspace.read(ws_name) as ws:
exp_set = ramble.experiment_set.ExperimentSet(ws)
expanded_app_context = ramble.context.Context()
expanded_app_context.context_name = "expanded_foms"
expanded_app_context.variables = {
"processes_per_node": "1",
"mpi_command": "",
"batch_submit": "",
}
expanded_wl_context = ramble.context.Context()
expanded_wl_context.context_name = "test_wl"
expanded_wl_context.variables = {}
expanded_exp_context = ramble.context.Context()
expanded_exp_context.context_name = "test"
expanded_exp_context.variables = {"n_ranks": "1", "my_var": "2.0"}
expanded_exp_context.is_template = True
exp_set.set_application_context(expanded_app_context)
exp_set.set_workload_context(expanded_wl_context)
exp_set.set_experiment_context(expanded_exp_context)
application_context = ramble.context.Context()
application_context.context_name = "basic"
application_context.variables = {
"app_var1": "1",
"app_var2": "2",
"processes_per_node": "1",
"mpi_command": "",
"batch_submit": "",
}
workload_context = ramble.context.Context()
workload_context.context_name = "test_wl"
workload_context.variables = {
"wl_var1": "1",
"wl_var2": "2",
}
experiment_context = ramble.context.Context()
experiment_context.context_name = "test1"
experiment_context.variables = {"n_ranks": "2", "my_var": "5.0"}
experiment_context.chained_experiments = [
{
"name": "expanded_foms.test_wl.test",
"order": "after_root",
"command": "{execute_experiment}",
"inherit_variables": ["my_var"],
},
]
exp_set.set_application_context(application_context)
exp_set.set_workload_context(workload_context)
exp_set.set_experiment_context(experiment_context)
exp_set.build_experiment_chains()
assert "basic.test_wl.test1" in exp_set.experiments
assert (
"basic.test_wl.test1.chain.0.expanded_foms.test_wl.test" in exp_set.chained_experiments
)
app_inst = exp_set.get_experiment("basic.test_wl.test1.chain.0.expanded_foms.test_wl.test")
assert app_inst.variables["my_var"] == "5.0"