Add stepwise intermediate reward for RL

CHANGELOG.md

@@ -11,6 +11,8 @@ This project adheres to [Semantic Versioning], with the exception that minor rel
 
 ### Changed
 
+- ✨ Improve RL reward design by adding intermediate rewards ([#526]) ([**@Shaobo-Zhou])
+- 🔧 Changed test circuit level for RL predictor from ALG to INDEP ([#449]) ([**@Shaobo-Zhou])
 - ✨ Remove support for custom names of trained models ([#489]) ([**@bachase**])
 - 🔥 Drop support for x86 macOS systems ([#421]) ([**@denialhaag**])
 
@@ -44,6 +46,8 @@ _📚 Refer to the [GitHub Release Notes](https://github.com/munich-quantum-tool
 
 <!-- PR links -->
 
+[#526]: https://github.com/munich-quantum-toolkit/predictor/pull/526
+[#449]: https://github.com/munich-quantum-toolkit/predictor/pull/449
 [#489]: https://github.com/munich-quantum-toolkit/predictor/pull/489
 [#421]: https://github.com/munich-quantum-toolkit/predictor/pull/421
 [#406]: https://github.com/munich-quantum-toolkit/predictor/pull/406
@@ -61,6 +65,7 @@ _📚 Refer to the [GitHub Release Notes](https://github.com/munich-quantum-tool
 [**@flowerthrower**]: https://github.com/flowerthrower
 [**@denialhaag**]: https://github.com/denialhaag
 [**@bachase**]: https://github.com/bachase
+[**@Shaobo-Zhou**]: https://github.com/Shaobo-Zhou
 
 <!-- General links -->
 

docs/setup.md

@@ -109,7 +109,7 @@ After setup, any quantum circuit can be compiled for the most suitable device wi
 from mqt.predictor import qcompile
 from mqt.bench import get_benchmark, BenchmarkLevel
 
-uncompiled_qc = get_benchmark("ghz", level=BenchmarkLevel.ALG, circuit_size=5)
+uncompiled_qc = get_benchmark("ghz", level=BenchmarkLevel.INDEP, circuit_size=5)
 compiled_qc, compilation_info, selected_device = qcompile(
     uncompiled_qc, figure_of_merit="expected_fidelity"
 )

pyproject.toml

@@ -94,6 +94,9 @@ Issues = "https://github.com/munich-quantum-toolkit/predictor/issues"
 Discussions = "https://github.com/munich-quantum-toolkit/predictor/discussions"
 Research = "https://www.cda.cit.tum.de/research/quantum/"
 
+[tool.check-sdist]
+sdist-only = ["src/mqt/predictor/_version.py"]
+
 [tool.hatch.build.targets.wheel]
 packages = ["src/mqt"]
 
@@ -124,6 +127,8 @@ filterwarnings = [
     'ignore:.*qiskit.providers.models is deprecated since Qiskit 1.2*:DeprecationWarning:',
     'ignore:.*The class ``qiskit.qobj.*`` is deprecated as of Qiskit 1.3.*:DeprecationWarning:',
     'ignore:.*The property ``qiskit.circuit.instruction.Instruction.*`` is deprecated as of qiskit 1.3.0.*:DeprecationWarning:',
+    "ignore:.*No canonical cost table defined for device.*:UserWarning:",
+
 ]
 
 

src/mqt/predictor/ml/predictor.py

@@ -61,6 +61,8 @@
 
 logger = logging.getLogger("mqt-predictor")
 
+NO_PARALLEL = sys.platform == "win32" and sys.version_info >= (3, 13)
+
 
 def setup_device_predictor(
     devices: list[Target],
@@ -227,7 +229,10 @@ def compile_training_circuits(
             with zipfile.ZipFile(str(path_zip), "r") as zip_ref:
                 zip_ref.extractall(path_uncompiled_circuits)
 
-        Parallel(n_jobs=num_workers, verbose=100)(
+        # On Windows + Python 3.13, joblib's default "loky" process backend is broken
+        # (missing `_posixsubprocess`). Fall back to no multiprocessing.
+        num_jobs = 1 if NO_PARALLEL else num_workers
+        Parallel(n_jobs=num_jobs, verbose=100)(
             delayed(self._compile_all_circuits_devicewise)(
                 device, timeout, path_uncompiled_circuits, path_compiled_circuits, logger.level
             )
@@ -267,7 +272,8 @@ def generate_training_data(
         names_list = []
         scores_list = []
 
-        results = Parallel(n_jobs=num_workers, verbose=100)(
+        num_jobs = 1 if NO_PARALLEL else num_workers
+        results = Parallel(n_jobs=num_jobs, verbose=100)(
             delayed(self._generate_training_sample)(
                 filename.name,
                 path_uncompiled_circuits,
@@ -276,6 +282,7 @@ def generate_training_data(
             )
             for filename in path_uncompiled_circuits.glob("*.qasm")
         )
+
         for sample in results:
             training_sample, circuit_name, scores = sample
             if all(score == -1 for score in scores):
@@ -405,8 +412,10 @@ def train_random_forest_model(
         if not training_data:
             training_data = self._get_prepared_training_data()
         num_cv = min(len(training_data.y_train), 5)
-        mdl = GridSearchCV(mdl, tree_param, cv=num_cv, n_jobs=8).fit(training_data.X_train, training_data.y_train)
-
+        num_jobs = 1 if NO_PARALLEL else 8
+        mdl = GridSearchCV(mdl, tree_param, cv=num_cv, n_jobs=num_jobs).fit(
+            training_data.X_train, training_data.y_train
+        )
         joblib_dump(mdl, save_mdl_path)
         logger.info("Random Forest model is trained and saved.")
 

src/mqt/predictor/reward.py

@@ -203,12 +203,15 @@ def estimated_success_probability(qc: QuantumCircuit, device: Target, precision:
                 if first_qubit_idx not in active_qubits:
                     continue
 
+                dt = device.dt or 1.0  # discrete time unit; fallback to 1.0 if unavailable
                 res *= np.exp(
                     -instruction.duration
+                    * dt
                     / min(device.qubit_properties[first_qubit_idx].t1, device.qubit_properties[first_qubit_idx].t2)
                 )
                 continue
             res *= 1 - device[gate_type][first_qubit_idx,].error
+
         else:
             second_qubit_idx = calc_qubit_index(qargs, qc.qregs, 1)
             res *= 1 - device[gate_type][first_qubit_idx, second_qubit_idx].error