Refactor default path handling to use task-type configuration with optional path parameter and optimized API calls

.github/workflows/tidy3d-python-client-tests.yml

@@ -262,7 +262,7 @@ jobs:
           BRANCH_NAME="${STEPS_EXTRACT_BRANCH_NAME_OUTPUTS_BRANCH_NAME}"
           echo $BRANCH_NAME
           # Allow only Jira keys from known projects, even if the branch has an author prefix
-          ALLOWED_JIRA_PROJECTS=("FXC" "SCEM")
+          ALLOWED_JIRA_PROJECTS=("FXC" "SCEM" "SCRF")
           JIRA_PROJECT_PATTERN=$(IFS='|'; echo "${ALLOWED_JIRA_PROJECTS[*]}")
           JIRA_PATTERN="(${JIRA_PROJECT_PATTERN})-[0-9]+"
 

docs/api/submit_simulations.rst

@@ -94,7 +94,6 @@ Information Containers
    :template: module.rst
 
    tidy3d.web.core.task_info.TaskInfo
-   tidy3d.web.core.task_info.TaskStatus
 
 
 Mode Solver Web API

tests/test_plugins/test_array_factor.py

@@ -363,16 +363,15 @@ def make_antenna_sim():
         remove_dc_component=False,  # Include DC component for more accuracy at low frequencies
     )
 
-    sim_unit = list(modeler.sim_dict.values())[0]
-
-    return sim_unit
+    return modeler
 
 
 def test_rectangular_array_calculator_array_make_antenna_array():
     """Test automatic antenna array creation."""
     freq0 = 10e9
     wavelength0 = td.C_0 / 10e9
-    sim_unit = make_antenna_sim()
+    modeler = make_antenna_sim()
+    sim_unit = list(modeler.sim_dict.values())[0]
     array_calculator = mw.RectangularAntennaArrayCalculator(
         array_size=(1, 2, 3),
         spacings=(0.5 * wavelength0, 0.6 * wavelength0, 0.4 * wavelength0),
@@ -437,8 +436,9 @@ def test_rectangular_array_calculator_array_make_antenna_array():
     assert len(sim_array.sources) == 6
 
     # check that override_structures are duplicated
-    assert len(sim_unit.grid_spec.override_structures) == 2
-    assert len(sim_array.grid_spec.override_structures) == 7
+    # assert len(sim_unit.grid_spec.override_structures) == 2
+    # assert len(sim_array.grid_spec.override_structures) == 7
+    assert sim_unit.grid.boundaries == modeler.base_sim.grid.boundaries
 
     # check that phase shifts are applied correctly
     phases_expected = array_calculator._antenna_phases
@@ -674,7 +674,8 @@ def test_rectangular_array_calculator_simulation_data_from_array_factor():
         phase_shifts=(np.pi / 3, np.pi / 4, np.pi / 5),
     )
 
-    sim_unit = make_antenna_sim()
+    modeler = make_antenna_sim()
+    sim_unit = list(modeler.sim_dict.values())[0]
 
     monitor = sim_unit.monitors[0]
     monitor_directivity = sim_unit.monitors[2]

tidy3d/components/mode/mode_solver.py

@@ -2655,7 +2655,7 @@ def _validate_modes_size(self) -> None:
                 "frequencies or modes."
             )
 
-    def validate_pre_upload(self, source_required: bool = True) -> None:
+    def validate_pre_upload(self) -> None:
         """Validate the fully initialized mode solver is ok for upload to our servers."""
         self._validate_modes_size()
 

tidy3d/components/mode/simulation.py

@@ -612,8 +612,8 @@ def plot_pml_mode_plane(
         """
         return self._mode_solver.plot_pml(ax=ax)
 
-    def validate_pre_upload(self, source_required: bool = False) -> None:
+    def validate_pre_upload(self) -> None:
         super().validate_pre_upload()
-        self._mode_solver.validate_pre_upload(source_required=source_required)
+        self._mode_solver.validate_pre_upload()
 
     _boundaries_for_zero_dims = validate_boundaries_for_zero_dims(warn_on_change=False)

tidy3d/components/tcad/mesher.py

@@ -24,3 +24,6 @@ class VolumeMesher(Tidy3dBaseModel):
 
     def _get_simulation_types(self) -> list[TCADAnalysisTypes]:
         return [TCADAnalysisTypes.MESH]
+
+    def validate_pre_upload(self):
+        return

tidy3d/plugins/smatrix/component_modelers/base.py

@@ -343,5 +343,9 @@ def run(
         )
         return data.smatrix()
 
+    def validate_pre_upload(self):
+        """Validate the modeler before upload."""
+        self.base_sim.validate_pre_upload(source_required=False)
+
 
 AbstractComponentModeler.update_forward_refs()

tidy3d/plugins/smatrix/component_modelers/modal.py

@@ -59,6 +59,11 @@ class ModalComponentModeler(AbstractComponentModeler):
         "by ``element_mappings``, the simulation corresponding to this column is skipped automatically.",
     )
 
+    @property
+    def base_sim(self):
+        """The base simulation."""
+        return self.simulation
+
     @cached_property
     def sim_dict(self) -> SimulationMap:
         """Generates all :class:`.Simulation` objects for the S-matrix calculation.

tidy3d/plugins/smatrix/component_modelers/terminal.py

@@ -223,15 +223,17 @@ def _warn_refactor_2_10(cls, values):
 
     @property
     def _sim_with_sources(self) -> Simulation:
-        """Instance of :class:`.Simulation` with all sources and absorbers added for each port, for troubleshooting."""
+        """Instance of :class:`.Simulation` with all sources and absorbers added for each port, for plotting."""
 
         sources = [port.to_source(self._source_time) for port in self.ports]
         absorbers = [
             port.to_absorber()
             for port in self.ports
             if isinstance(port, WavePort) and port.absorber
         ]
-        return self.simulation.updated_copy(sources=sources, internal_absorbers=absorbers)
+        return self.simulation.updated_copy(
+            sources=sources, internal_absorbers=absorbers, validate=False
+        )
 
     @equal_aspect
     @add_ax_if_none
@@ -382,18 +384,17 @@ def matrix_indices_run_sim(self) -> tuple[NetworkIndex, ...]:
     def sim_dict(self) -> SimulationMap:
         """Generate all the :class:`.Simulation` objects for the port parameter calculation."""
 
+        # Check base simulation for grid size at ports
+        TerminalComponentModeler._check_grid_size_at_ports(self.base_sim, self._lumped_ports)
+        TerminalComponentModeler._check_grid_size_at_wave_ports(self.base_sim, self._wave_ports)
+
         sim_dict = {}
         # Now, create simulations with wave port sources and mode solver monitors for computing port modes
         for network_index in self.matrix_indices_run_sim:
             task_name, sim_with_src = self._add_source_to_sim(network_index)
             # update simulation
             sim_dict[task_name] = sim_with_src
 
-        # Check final simulations for grid size at ports
-        for _, sim in sim_dict.items():
-            TerminalComponentModeler._check_grid_size_at_ports(sim, self._lumped_ports)
-            TerminalComponentModeler._check_grid_size_at_wave_ports(sim, self._wave_ports)
-
         return SimulationMap(keys=tuple(sim_dict.keys()), values=tuple(sim_dict.values()))
 
     @cached_property
@@ -414,7 +415,10 @@ def _base_sim_no_radiation_monitors(self) -> Simulation:
 
         # Make an initial simulation with new grid_spec to determine where LumpedPorts are snapped
         sim_wo_source = self.simulation.updated_copy(
-            grid_spec=grid_spec, lumped_elements=lumped_resistors
+            grid_spec=grid_spec,
+            lumped_elements=lumped_resistors,
+            validate=False,
+            deep=False,
         )
         snap_centers = {}
         for port in self._lumped_ports:
@@ -480,7 +484,11 @@ def _base_sim_no_radiation_monitors(self) -> Simulation:
             )
 
         # update base simulation with updated set of shared components
-        sim_wo_source = sim_wo_source.copy(update=update_dict)
+        sim_wo_source = sim_wo_source.updated_copy(
+            **update_dict,
+            validate=False,
+            deep=False,
+        )
 
         # extrude port structures
         sim_wo_source = self._extrude_port_structures(sim=sim_wo_source)
@@ -527,7 +535,10 @@ def base_sim(self) -> Simulation:
         """The base simulation with all components added, including radiation monitors."""
         base_sim_tmp = self._base_sim_no_radiation_monitors
         mnts_with_radiation = list(base_sim_tmp.monitors) + list(self._finalized_radiation_monitors)
-        return base_sim_tmp.updated_copy(monitors=mnts_with_radiation)
+        grid_spec = GridSpec.from_grid(base_sim_tmp.grid)
+        grid_spec.attrs["from_grid_spec"] = base_sim_tmp.grid_spec
+        # We skipped validations up to now, here we finally validate the base sim
+        return base_sim_tmp.updated_copy(monitors=mnts_with_radiation, grid_spec=grid_spec)
 
     def _generate_radiation_monitor(
         self, simulation: Simulation, auto_spec: DirectivityMonitorSpec
@@ -712,7 +723,10 @@ def _add_source_to_sim(self, source_index: NetworkIndex) -> tuple[str, Simulatio
             )
         task_name = self.get_task_name(port=port, mode_index=mode_index)
 
-        return (task_name, self.base_sim.updated_copy(sources=[port_source]))
+        return (
+            task_name,
+            self.base_sim.updated_copy(sources=[port_source], validate=False, deep=False),
+        )
 
     @cached_property
     def _source_time(self):
@@ -958,6 +972,8 @@ def _extrude_port_structures(self, sim: Simulation) -> Simulation:
             sim = sim.updated_copy(
                 grid_spec=GridSpec.from_grid(sim.grid),
                 structures=[*sim.structures, *all_new_structures],
+                validate=False,
+                deep=False,
             )
 
         return sim

tidy3d/web/__init__.py

@@ -30,7 +30,6 @@
     load,
     load_simulation,
     monitor,
-    postprocess_start,
     real_cost,
     start,
     test,
@@ -58,7 +57,6 @@
     "load",
     "load_simulation",
     "monitor",
-    "postprocess_start",
     "real_cost",
     "run",
     "run_async",

tidy3d/web/api/container.py

@@ -389,25 +389,7 @@ def status(self) -> str:
         """Return current status of :class:`Job`."""
         if self.load_if_cached:
             return "success"
-        if web._is_modeler_batch(self.task_id):
-            detail = self.get_info()
-            status = detail.totalStatus.value
-            return status
-        else:
-            return self.get_info().status
-
-    @property
-    def postprocess_status(self) -> Optional[str]:
-        """Return current postprocess status of :class:`Job` if it is a Component Modeler."""
-        if web._is_modeler_batch(self.task_id):
-            detail = self.get_info()
-            return detail.postprocessStatus
-        else:
-            log.warning(
-                f"Task ID '{self.task_id}' is not a modeler batch job. "
-                "'postprocess_start' is only applicable to Component Modelers"
-            )
-            return
+        return self.get_info().status
 
     def start(self, priority: Optional[int] = None) -> None:
         """Start running a :class:`Job`.
@@ -548,37 +530,6 @@ def estimate_cost(self, verbose: bool = True) -> float:
             return 0.0
         return web.estimate_cost(self.task_id, verbose=verbose, solver_version=self.solver_version)
 
-    def postprocess_start(self, worker_group: Optional[str] = None, verbose: bool = True) -> None:
-        """
-        If the job is a modeler batch, checks if the run is complete and starts
-        the postprocess phase.
-
-        This function does not wait for postprocessing to finish and is only
-        applicable to Component Modeler batch jobs.
-
-        Parameters
-        ----------
-        worker_group : Optional[str] = None
-            The specific worker group to run the postprocessing task on.
-        verbose : bool = True
-            Whether to print info messages. This overrides the Job's 'verbose' setting for this call.
-        """
-        # First, confirm that the task is a modeler batch job.
-        if not web._is_modeler_batch(self.task_id):
-            # If not, inform the user and exit.
-            # This warning is important and should not be suppressed.
-            log.warning(
-                f"Task ID '{self.task_id}' is not a modeler batch job. "
-                "'postprocess_start' is only applicable to Component Modelers"
-            )
-            return
-
-        # If it is a modeler batch, call the dedicated function to start postprocessing.
-        # The verbosity is a combination of the job's setting and the method's parameter.
-        web.postprocess_start(
-            batch_id=self.task_id, verbose=(self.verbose and verbose), worker_group=worker_group
-        )
-
     @staticmethod
     def _check_path_dir(path: PathLike) -> None:
         """Make sure parent directory of ``path`` exists and create it if not.
@@ -1043,21 +994,6 @@ def get_run_info(self) -> dict[TaskName, RunInfo]:
             run_info_dict[task_name] = run_info
         return run_info_dict
 
-    def postprocess_start(self, worker_group: Optional[str] = None, verbose: bool = True) -> None:
-        """
-        Start the postprocess phase for all applicable jobs in the batch.
-
-        This simply forwards to each Job's `postprocess_start(...)`. The Job decides
-        whether it's a Component Modeler task and whether it can/should start now.
-        This method does not wait for postprocessing to finish.
-        """
-        if self.verbose and verbose:
-            console = get_logging_console()
-            console.log("Attempting to start postprocessing for jobs in the batch.")
-
-        for job in self.jobs.values():
-            job.postprocess_start(worker_group=worker_group, verbose=verbose)
-
     def monitor(
         self,
         *,
@@ -1069,7 +1005,6 @@ def monitor(
         """
         Monitor progress of each running task.
 
-        - For Component Modeler jobs, automatically triggers postprocessing once run finishes.
         - Optionally downloads results as soon as a job reaches final success.
         - Rich progress bars in verbose mode; quiet polling otherwise.
 
@@ -1095,16 +1030,8 @@ def monitor(
             self._check_path_dir(path_dir=path_dir)
             download_executor = ThreadPoolExecutor(max_workers=self.num_workers)
 
-        def _should_download(job: Job) -> bool:
-            status = job.status
-            if not web._is_modeler_batch(job.task_id):
-                return status == "success"
-            if status == "success":
-                return True
-            return status == "run_success" and getattr(job, "postprocess_status", None) == "success"
-
         def schedule_download(job: Job) -> None:
-            if download_executor is None or not _should_download(job):
+            if download_executor is None or job.status not in COMPLETED_STATES:
                 return
             task_id = job.task_id
             if task_id in downloads_started:
@@ -1128,12 +1055,7 @@ def schedule_download(job: Job) -> None:
         def check_continue_condition(job: Job) -> bool:
             if job.load_if_cached:
                 return False
-            status = job.status
-            if not web._is_modeler_batch(job.task_id):
-                return status not in END_STATES
-            if status == "run_success":
-                return job.postprocess_status not in END_STATES
-            return status not in END_STATES
+            return job.status not in END_STATES
 
         def pbar_description(
             task_name: str, status: str, max_name_length: int, status_width: int
@@ -1157,9 +1079,6 @@ def pbar_description(
         max_task_name = max(len(task_name) for task_name in self.jobs.keys())
         max_name_length = min(30, max(max_task_name, 15))
 
-        # track which modeler jobs we've already kicked into postprocess
-        postprocess_started_tasks: set[str] = set()
-
         try:
             console = None
             progress_columns = []
@@ -1195,17 +1114,6 @@ def pbar_description(
                     for task_name, job in self.jobs.items():
                         status = job.status
 
-                        # auto-start postprocess for modeler jobs when run finishes
-                        if (
-                            web._is_modeler_batch(job.task_id)
-                            and status == "run_success"
-                            and job.task_id not in postprocess_started_tasks
-                        ):
-                            job.postprocess_start(
-                                worker_group=postprocess_worker_group, verbose=True
-                            )
-                            postprocess_started_tasks.add(job.task_id)
-
                         schedule_download(job)
 
                         if self.verbose: