Merge pull request #1001 from bashtage/fixes-for-pandas-3

MAINT: Fix PDF for pandas 3 changes

Author: bashtage

ci/azure/azure_template_posix.yml

@@ -16,19 +16,22 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      python38_legacy:
-        python.version: '3.8'
-        PANDAS: 1.5.3
       python39_legacy:
         python.version: '3.9'
         PANDAS: 1.5.3
-      python310_recent:
-        python.version: '3.9'
+        NUMPY: 1.23.4
+      python310_legacy:
+        python.version: '3.10'
         PANDAS: 2.0.3
+        NUMPY: 1.24.4
       python311_latest:
         python.version: '3.11'
+        PANDAS: 2.1.4
+        NUMPY: 1.26.4
       python312_latest:
         python.version: '3.12'
+      python313_latest:
+        python.version: '3.13'
 
   steps:
   - task: UsePythonVersion@0
@@ -40,6 +43,9 @@ jobs:
       python -m pip install pip setuptools -U      
       python -m pip install -r requirements.txt
       python -m pip install -r requirements-dev.txt
+      if [[ -n ${NUMPY} ]]; then 
+        python -m pip install numpy==${NUMPY}
+      fi;
       if [[ -n ${PANDAS} ]]; then 
         python -m pip install pandas==${PANDAS}
       fi;
@@ -73,9 +79,8 @@ jobs:
       testRunTitle: 'Python $(python.version)'
     condition: succeededOrFailed()
 
-  - task: PublishCodeCoverageResults@1
+  - task: PublishCodeCoverageResults@2
     inputs:
-      codeCoverageTool: Cobertura
       summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
     condition: eq(variables['coverage'], 'true')
 

ci/azure/azure_template_windows.yml

@@ -7,7 +7,6 @@ parameters:
   # defaults for any parameters that are not specified
   name: ''
   vmImage: ''
-  coverage: true
 
 jobs:
 
@@ -16,19 +15,21 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      python38_legacy:
-        python.version: '3.8'
-        PANDAS: 1.5.3
       python39_legecy:
         python.version: '3.9'
         PANDAS: 1.5.3
+        NUMPY: 1.23.4
       python310_recent:
         python.version: '3.10'
-        PANDAS: 2.0.3
+        NUMPY: 1.24.4
       python311_latest:
         python.version: '3.11'
+        PANDAS: 2.1.4
+        NUMPY: 1.26.4
       python312_latest:
         python.version: '3.12'
+      python313_latest:
+        python.version: '3.13'
 
   steps:
   - task: UsePythonVersion@0
@@ -44,10 +45,13 @@ jobs:
     displayName: 'Install dependencies'
 
   - powershell: |
+      if ($null -ne $env:NUMPY) {
+        python -m pip install numpy==$env:NUMPY
+      }
       if ($null -ne $env:PANDAS) {
-        python -m pip install pandas==${PANDAS}
+        python -m pip install pandas==$env:PANDAS
       }
-    displayName: 'Update pandas (if needed)'
+    displayName: 'Update numpy and pandas (if needed)'
 
   - script: |
       python -m pip install -e . -vv
@@ -64,25 +68,12 @@ jobs:
 
   - script: |
       echo "Testing editable install"
-      if [[ ${COVERAGE} == "true" ]]; then
-        export COVERAGE_OPTS="--cov-config .coveragerc --cov=pandas_datareader --cov-report xml:coverage.xml --cov-report term"
-      fi
-      echo pytest -m "${PYTEST_PATTERN}" --junitxml=junit/test-results.xml -n auto --durations=25 ${COVERAGE_OPTS} pandas_datareader/tests
-      pytest -m "${PYTEST_PATTERN}" --junitxml=junit/test-results.xml -n auto --durations=25 ${COVERAGE_OPTS} pandas_datareader/tests
+      echo pytest --durations=25 pandas_datareader/tests
+      pytest --durations=25 pandas_datareader/tests
     displayName: 'Run tests'
 
   - task: PublishTestResults@2
     inputs:
       testResultsFiles: '**/test-results.xml'
       testRunTitle: 'Python $(python.version)'
     condition: succeededOrFailed()
-
-  - task: PublishCodeCoverageResults@1
-    inputs:
-      codeCoverageTool: Cobertura
-      summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
-    condition: eq(variables['coverage'], 'true')
-
-  - bash: bash <(curl -s https://codecov.io/bash)
-    displayName: 'CodeCov upload'
-    condition: eq(variables['coverage'], 'true')

pandas_datareader/base.py

@@ -115,8 +115,9 @@ def _read_url_as_StringIO(self, url, params=None):
         if len(text) == 0:
             service = self.__class__.__name__
             raise OSError(
-                "{} request returned no data; check URL for invalid "
-                "inputs: {}".format(service, self.url)
+                "{} request returned no data; check URL for invalid inputs: {}".format(
+                    service, self.url
+                )
             )
         if isinstance(text, bytes):
             out.write(text.decode("utf-8"))

pandas_datareader/compat/__init__.py

@@ -1,4 +1,5 @@
 from io import StringIO
+import sys
 from urllib.error import HTTPError
 
 from pandas.api.types import is_list_like, is_number
@@ -12,7 +13,7 @@
     "assert_frame_equal",
     "is_list_like",
     "is_number",
-    "reduce",
+    "PYTHON_LT_3_10",
 ]
 
 
@@ -31,3 +32,6 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None, compression=None):
             filepath_or_buffer, encoding=encoding, compression=None
         )
         return tmp
+
+
+PYTHON_LT_3_10 = sys.version_info <= (3, 10)

pandas_datareader/data.py

@@ -683,8 +683,7 @@ def DataReader(
 def Options(symbol, data_source=None, session=None):
     if data_source is None:
         warnings.warn(
-            "Options(symbol) is deprecated, use Options(symbol,"
-            " data_source) instead",
+            "Options(symbol) is deprecated, use Options(symbol, data_source) instead",
             FutureWarning,
             stacklevel=2,
         )

pandas_datareader/famafrench.py

@@ -6,7 +6,7 @@
 from pandas import read_csv, to_datetime
 
 from pandas_datareader.base import _BaseReader
-from pandas_datareader.compat import StringIO
+from pandas_datareader.compat import PYTHON_LT_3_10, StringIO
 
 _URL = "http://mba.tuck.dartmouth.edu/pages/faculty/ken.french/"
 _URL_PREFIX = "ftp/"
@@ -57,7 +57,7 @@ def _read_zipfile(self, url):
         with tempfile.TemporaryFile() as tmpf:
             tmpf.write(raw)
             with ZipFile(tmpf, "r") as zf:
-                try:                
+                try:
                     data = zf.open(zf.namelist()[0]).read().decode("utf-8", "ignore")
                 except UnicodeDecodeError:
                     data = zf.open(zf.namelist()[0]).read().decode(encoding="cp1252")
@@ -78,8 +78,6 @@ def read(self):
     def _read_one_data(self, url, params):
         params = {
             "index_col": 0,
-            "parse_dates": [0],
-            "date_parser": _parse_date_famafrench,
         }
 
         # headers in these files are not valid
@@ -89,7 +87,12 @@ def _read_one_data(self, url, params):
             else:
                 c = ["Count"]
             r = list(range(0, 105, 5))
-            params["names"] = ["Date"] + c + list(zip(r, r[1:], strict=False))
+
+            if PYTHON_LT_3_10:
+                additional_params = list(zip(r, r[1:]))  # noqa: B905
+            else:
+                additional_params = list(zip(r, r[1:], strict=False))
+            params["names"] = ["Date"] + c + additional_params
 
             if self.symbols != "Prior_2-12_Breakpoints":
                 params["skiprows"] = 1
@@ -111,12 +114,14 @@ def _read_one_data(self, url, params):
             start = 0 if not match else match.start()
 
             df = read_csv(StringIO("Date" + src[start:]), **params)
-            try:
-                idx_name = df.index.name  # hack for pandas 0.16.2
-                df = df.to_period(df.index.inferred_freq[:1])
-                df.index.name = idx_name
-            except Exception:
-                pass
+            if df.index.min() > 190000:
+                df.index = to_datetime(df.index.astype(str), format="%Y%m").to_period(
+                    freq="M"
+                )
+            else:
+                df.index = to_datetime(df.index.astype(str), format="%Y").to_period(
+                    freq="Y"
+                )
             df = df.truncate(self.start, self.end)
             datasets[i] = df
 

pandas_datareader/fred.py

@@ -58,8 +58,13 @@ def fetch_data(url, name):
                     ) from exc
                 raise
 
+        try:
+            data = [fetch_data(url, n) for url, n in zip(urls, names, strict=True)]
+        except TypeError:
+            # Python 3.9 only
+            data = [fetch_data(url, n) for url, n in zip(urls, names)]  # noqa: B905
         df = concat(
-            [fetch_data(url, n) for url, n in zip(urls, names, strict=True)],
+            data,
             axis=1,
             join="outer",
         )

pandas_datareader/iex/daily.py

@@ -15,7 +15,6 @@
 
 
 class IEXDailyReader(_DailyBaseReader):
-
     """
     Returns DataFrame of historical stock prices
     from symbols, over date range, start to end. To avoid being penalized by

pandas_datareader/moex.py

@@ -197,8 +197,9 @@ def read_all_boards(self):
 
         if len(dfs) == 0:
             raise OSError(
-                "{} returned no data; "
-                "check URL or correct a date".format(self.__class__.__name__)
+                "{} returned no data; check URL or correct a date".format(
+                    self.__class__.__name__
+                )
             )
         elif len(dfs) > 1:
             b = pd.concat(dfs, axis=0, join="outer", sort=True)
@@ -226,8 +227,9 @@ def _read_url_as_String(self, url, params=None):
         if len(text) == 0:
             service = self.__class__.__name__
             raise OSError(
-                "{} request returned no data; check URL for invalid "
-                "inputs: {}".format(service, self.url)
+                "{} request returned no data; check URL for invalid inputs: {}".format(
+                    service, self.url
+                )
             )
         if isinstance(text, bytes):
             text = text.decode("windows-1251")

pandas_datareader/stooq.py

@@ -2,7 +2,6 @@
 
 
 class StooqDailyReader(_DailyBaseReader):
-
     """
     Returns DataFrame/dict of Dataframes of historical stock prices from
     symbols, over date range, start to end.