Skip to content

Improved and added to loops over tables #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
paulsaxe merged 1 commit into from
May 15, 2025
Merged

Improved and added to loops over tables #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions HISTORY.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
=======
History
=======
2025.5.15: Improved and added to loops over tables
* Added a 'between' operator to filter rows of tables.
* Ensured the correct types of variables in the rows of a table, fixing an issues
that caused e.g. integers to be treated as floats.
* Added an option, on by default, to make variables with the name of the columns in
the tables, holding the values.
* Using directory names that are the index of the row in the table.

2024.12.23: Bugfix: Directory name widget not displayed
* The directory name widget was not displayed for loops over systems in the database.
This is now corrected.
Expand Down
195 changes: 100 additions & 95 deletions loop_step/loop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,12 +297,6 @@ def run(self):
)
self._loop_value = 0
self._loop_length = self.table.shape[0]
printer.important(
__(
f"The loop will have {self._loop_length} iterations.\n\n",
indent=self.indent + 4 * " ",
)
)
if self.variable_exists("_loop_indices"):
tmp = self.get_variable("_loop_indices")
self.set_variable(
Expand All @@ -314,29 +308,91 @@ def run(self):
)
else:
self.set_variable("_loop_indices", (None,))
where = P["where"]
if where == "Use all rows":
pass
elif where == "Select rows where column":
column = P["query-column"]
op = P["query-op"]
value = P["query-value"]
if self.table.shape[0] > 0:
row = self.table.iloc[0]
tmp = pprint.pformat(row)
self.logger.debug(f"Row is\n{tmp}")
if column not in row:
for key in row.keys():
if column.lower() == key.lower():
column = key
break
if column not in row:
where = P["where"]
if where == "Use all rows":
table_indices = [*self.table.index]
n_table_indices = len(table_indices)
elif where == "Select rows where column":
tmp_col = P["query-column"].lower()
column = None
op = P["query-op"]

for col in self.table:
if col.lower() == tmp_col:
column = col
if column is None:
column = P["query-column"]
raise ValueError(
f"Looping over table with criterion on column '{column}': "
"that column does not exist."
)
else:
raise NotImplementedError(f"Loop cannot handle '{where}'")

dtype = self.table.dtypes[column]
value = dtype.type(P["query-value"])
value2 = dtype.type(P["query-value2"])

# Find the indices
table_indices = []
for i, row_value in zip(self.table.index, self.table[column]):
if op == "==":
if row_value == value:
table_indices.append(i)
elif op == "!=":
if row_value != value:
table_indices.append(i)
elif op == ">":
if row_value > value:
table_indices.append(i)
elif op == ">=":
if row_value >= value:
table_indices.append(i)
elif op == "<":
if row_value < value:
table_indices.append(i)
elif op == "<=":
if row_value <= value:
table_indices.append(i)
elif op == "between":
if row_value >= value and row_value <= value2:
table_indices.append(i)
elif op == "contains":
if value in row_value:
table_indices.append(i)
elif op == "does not contain":
if value not in row_value:
table_indices.append(i)
elif op == "contains regexp":
if re.search(value, row_value) is not None:
table_indices.append(i)
elif op == "does not contain regexp":
if re.search(value, row_value) is None:
table_indices.append(i)
elif op == "is empty":
# Might be numpy.nan, and NaN != NaN hence odd test.
if row_value == "" or row_value != row_value:
table_indices.append(i)
elif op == "is not empty":
if row_value != "" and row_value == row_value:
table_indices.append(i)
else:
raise NotImplementedError(
f"Loop query '{op}' not implemented"
)
n_table_indices = len(table_indices)
else:
raise NotImplementedError(f"Loop cannot handle '{where}'")
printer.important(
__(
f"The loop will have {n_table_indices} iterations.\n\n",
indent=self.indent + 4 * " ",
)
)

if n_table_indices > 0:
index = table_indices[0]
index_is_int = isinstance(index, int)
if index_is_int:
fmt = f"0{len(str(max(table_indices) + 1))}d"
elif P["type"] == "For systems in the database":
# Get a list of all the matching systems and configurations
system_db = self.get_variable("_system_db")
Expand Down Expand Up @@ -532,64 +588,8 @@ def run(self):
self.set_variable("_loop_index", self._loop_value)
self.logger.info(" Loop value = {}".format(value))
elif P["type"] == "For rows in table":
# Loop until query is satisfied
while True:
self._loop_value += 1

if self._loop_value > self.table.shape[0]:
break

if where == "Use all rows":
break

row = self.table.iloc[self._loop_value - 1]

self.logger.debug(f"Query {row[column]} {op} {value}")
_type = type(row[column])
_value = _type(value)
if op == "==":
if row[column] == _value:
break
elif op == "!=":
if row[column] != _value:
break
elif op == ">":
if row[column] > _value:
break
elif op == ">=":
if row[column] >= _value:
break
elif op == "<":
if row[column] < _value:
break
elif op == "<=":
if row[column] <= _value:
break
elif op == "contains":
if _value in row[column]:
break
elif op == "does not contain":
if _value not in row[column]:
break
elif op == "contains regexp":
if re.search(value, row[column]) is not None:
break
elif op == "does not contain regexp":
if re.search(value, row[column]) is None:
break
elif op == "is empty":
# Might be numpy.nan, and NaN != NaN hence odd test.
if row[column] == "" or row[column] != row[column]:
break
elif op == "is not empty":
if row[column] != "" and row[column] == row[column]:
break
else:
raise NotImplementedError(
f"Loop query '{op}' not implemented"
)

if self._loop_value > self.table.shape[0]:
self._loop_value += 1
if self._loop_value > n_table_indices:
self._loop_value = None

self.delete_variable("_row")
Expand Down Expand Up @@ -620,24 +620,29 @@ def run(self):

# Set up the index variables
self.logger.debug(" _loop_value = {}".format(self._loop_value))
index = table_indices[self._loop_value - 1]
tmp = self.get_variable("_loop_indices")
self.logger.debug(" _loop_indices = {}".format(tmp))
self.set_variable(
"_loop_indices",
(*tmp[0:-1], self.table.index[self._loop_value - 1]),
)
self.set_variable("_loop_indices", (*tmp[0:-1], index))
self.logger.debug(
" --> {}".format(self.get_variable("_loop_indices"))
)
self.set_variable(
"_loop_index", self.table.index[self._loop_value - 1]
)
self.table_handle["current index"] = self.table.index[
self._loop_value - 1
]
self.set_variable("_loop_index", index)
self.table_handle["current index"] = index

# Name of directory is the index (+1 since tends to be 0 based)
if index_is_int:
self._custom_directory_name = f"iter_{index + 1:{fmt}}"
else:
self._custom_directory_name = self.safe_filename(str(index))

row = self.table.iloc[self._loop_value - 1]
row = {k: self.table.at[index, k] for k in self.table}
self.set_variable("_row", row)
if P["as variables"]:
for key, value in row.items():
# Make a safe variable name
key = re.sub(r"[-\\ / \+\*()]", "_", key)
self.set_variable(key, value)
self.logger.debug(" _row = {}".format(row))
elif P["type"] == "For systems in the database":
self._loop_value += 1
Expand Down Expand Up @@ -725,7 +730,7 @@ def run(self):
next_node = self
except SkipIteration:
next_node = self
shutil.rmtree(iter_dir)
shutil.rmtree(iter_dir, ignore_errors=True)
except Exception as e:
tmp = self.working_path.name
printer.job(f"Caught exception in loop iteration {tmp}: {str(e)}")
Expand Down
22 changes: 20 additions & 2 deletions loop_step/loop_parameters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# -*- coding: utf-8 -*-
"""Control parameters for loops
"""
"""Control parameters for loops"""

import logging
import seamm
Expand Down Expand Up @@ -109,6 +108,7 @@ class LoopParameters(seamm.Parameters):
">=",
"<",
"<=",
"between",
"contains",
"does not contain",
"contains regexp",
Expand All @@ -129,6 +129,24 @@ class LoopParameters(seamm.Parameters):
"description": "",
"help_text": ("Value to use in the test"),
},
"query-value2": {
"default": "",
"kind": "string",
"default_units": "",
"enumeration": tuple(),
"format_string": "s",
"description": "",
"help_text": "The second value to use in the test",
},
"as variables": {
"default": "yes",
"kind": "boolean",
"default_units": "",
"enumeration": ("yes", "no"),
"format_string": "",
"description": "Values as variables:",
"help_text": "Whether to put the values for the row as seperate variables.",
Comment on lines +142 to +148

Copilot AI May 15, 2025

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The parameter "as variables" is declared with kind "boolean" but uses string values "yes"/"no" in its enumeration. Either change kind to "enumeration" or use actual boolean values (true/false) to match the declared kind.

Suggested change
"default": "yes",
"kind": "boolean",
"default_units": "",
"enumeration": ("yes", "no"),
"format_string": "",
"description": "Values as variables:",
"help_text": "Whether to put the values for the row as seperate variables.",
"default": True,
"kind": "boolean",
"default_units": "",
"enumeration": (True, False),
"format_string": "",
"description": "Values as variables:",
"help_text": "Whether to put the values for the row as separate variables.",

Copilot uses AI. Check for mistakes.
},
"where system name": {
"default": "is anything",
"kind": "string",
Expand Down
5 changes: 5 additions & 0 deletions loop_step/tk_loop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,8 @@ def reset_dialog(self, widget=None):
frame.columnconfigure(3, weight=0)
self["table"].grid(row=row, column=2, columnspan=2, sticky=tk.EW)
row += 1
self["as variables"].grid(row=row, column=1, columnspan=2, sticky=tk.EW)
row += 1
self["where"].grid(row=row, column=1, columnspan=2, sticky=tk.EW)
where = self["where"].get()
if where != "Use all rows":
Expand All @@ -212,6 +214,9 @@ def reset_dialog(self, widget=None):
if "empty" not in op:
self["query-value"].grid(row=row, column=5, sticky=tk.EW)
frame.columnconfigure(5, weight=1)
if op == "between":
self["query-value2"].grid(row=row, column=6, sticky=tk.EW)
frame.columnconfigure(6, weight=1)
row += 1
elif loop_type == "For systems in the database":
row += 1
Expand Down