feat: add the ignore user_agent ability

Author: Kl0ven

controller/sentry/admin.py

@@ -222,6 +222,7 @@ class AppAdmin(
                 "fields": (
                     "wsgi_collect_metrics",
                     "wsgi_ignore_path",
+                    "wsgi_ignore_user_agent",
                     "wsgi_metrics",
                 ),
             },

controller/sentry/metrics/celery.py

@@ -1,16 +1,35 @@
 """Celery Metrics."""
 from collections import Counter
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Union
+
+from controller.sentry.utils import depth
 
 if TYPE_CHECKING:  # pragma: no cover
     from controller.sentry.models import App
 
 
-def celery_merger(app: "App", new: dict[str, int]) -> None:
+def celery_merger(app: "App", new: Union[dict[str, int], dict[str, dict[str, int]]]) -> None:
     """celery_merger function is used to merge :attr:`Celery <controller.sentry.choices.MetricType.CELERY>` metrics.
 
     Args:
         app (App): The app associated to the metric
         new (dict[str, int]): The new metric dict
     """
-    app.celery_metrics = dict(Counter(app.celery_metrics) + Counter(new))
+    # Code for Migration
+    if depth(app.celery_metrics) == 1:
+        app.celery_metrics = {"task": app.celery_metrics}
+
+    if depth(new) == 1:
+        new = {"task": new}
+    # End of migration code
+
+    if app.celery_metrics is None:
+        app.celery_metrics = {}
+
+    tmp = {}
+    for key in set(list(app.celery_metrics.keys()) + list(new.keys())):
+        old_value = app.celery_metrics.get(key)
+        new_value = new.get(key)
+        tmp[key] = dict(Counter(old_value) + Counter(new_value))
+
+    app.celery_metrics = tmp

controller/sentry/metrics/wsgi.py

@@ -1,16 +1,35 @@
 """Wsgi Metrics."""
 from collections import Counter
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Union
+
+from controller.sentry.utils import depth
 
 if TYPE_CHECKING:  # pragma: no cover
     from controller.sentry.models import App
 
 
-def wsgi_merger(app: "App", new: dict[str, int]) -> None:
+def wsgi_merger(app: "App", new: Union[dict[str, int], dict[str, dict[str, int]]]) -> None:
     """wsgi_merger function is used to merge :attr:`WSGI <controller.sentry.choices.MetricType.WSGI>` metrics.
 
     Args:
         app (App): The app associated to the metric
         new (dict[str, int]): The new metric dict
     """
-    app.wsgi_metrics = dict(Counter(app.wsgi_metrics) + Counter(new))
+    # Code for Migration
+    if depth(app.wsgi_metrics) == 1:
+        app.wsgi_metrics = {"path": app.wsgi_metrics}
+
+    if depth(new) == 1:
+        new = {"path": new}
+    # End of migration code
+
+    if app.wsgi_metrics is None:
+        app.wsgi_metrics = {}
+
+    tmp = {}
+    for key in set(list(app.wsgi_metrics.keys()) + list(new.keys())):
+        old_value = app.wsgi_metrics.get(key)
+        new_value = new.get(key)
+        tmp[key] = dict(Counter(old_value) + Counter(new_value))
+
+    app.wsgi_metrics = tmp

controller/sentry/migrations/0015_app_wsgi_ignore_user_agent_and_more.py

@@ -0,0 +1,43 @@
+# Generated by Django 4.2 on 2023-05-28 14:20
+
+import functools
+
+import django_better_admin_arrayfield.models.fields
+from django.db import migrations, models
+
+import controller.sentry.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("sentry", "0014_alter_app_celery_collect_metrics_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="app",
+            name="wsgi_ignore_user_agent",
+            field=django_better_admin_arrayfield.models.fields.ArrayField(
+                base_field=models.CharField(blank=True, max_length=50),
+                blank=True,
+                default=functools.partial(
+                    controller.sentry.models.settings_default_value, *("DEFAULT_WSGI_IGNORE_USER_AGENT",), **{}
+                ),
+                help_text="A list of user agent to ignore, matched with startswith",
+                size=None,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="app",
+            name="wsgi_ignore_path",
+            field=django_better_admin_arrayfield.models.fields.ArrayField(
+                base_field=models.CharField(blank=True, max_length=50),
+                blank=True,
+                default=functools.partial(
+                    controller.sentry.models.settings_default_value, *("DEFAULT_WSGI_IGNORE_PATHS",), **{}
+                ),
+                help_text="A list of path to ignore, matched using fill match ==",
+                size=None,
+            ),
+        ),
+    ]

controller/sentry/models.py

@@ -77,6 +77,16 @@ class App(models.Model):
         models.CharField(max_length=50, blank=True),
         blank=True,
         default=partial(settings_default_value, "DEFAULT_WSGI_IGNORE_PATHS"),
+        help_text="A list of path to ignore, matched using fill match ==",
+    )
+    wsgi_ignore_user_agent = ArrayField(
+        models.CharField(max_length=50, blank=True),
+        blank=True,
+        default=partial(
+            settings_default_value,
+            "DEFAULT_WSGI_IGNORE_USER_AGENT",
+        ),
+        help_text="A list of user agent to ignore, matched with startswith",
     )
     wsgi_collect_metrics = models.BooleanField(default=False, verbose_name="wsgi metrics")
     wsgi_metrics = models.JSONField(null=True, blank=True)

controller/sentry/serializers.py

@@ -17,6 +17,7 @@ class Meta:
             "active_sample_rate",
             "active_window_end",
             "wsgi_ignore_path",
+            "wsgi_ignore_user_agent",
             "wsgi_collect_metrics",
             "celery_ignore_task",
             "celery_collect_metrics",

controller/sentry/tests/test_models.py

@@ -13,4 +13,4 @@ def test_app_model_merge():
     app.merge({"type": MetricType.CELERY, "data": {"test": 1}})
     collect, metrics = app.get_metric(MetricType.CELERY)
     assert not collect
-    assert metrics == {"test1": 5, "test": 6}
+    assert metrics == {"task": {"test1": 5, "test": 6}}

controller/sentry/tests/test_views.py

@@ -1,3 +1,4 @@
+from collections import Counter
 from unittest.mock import Mock, patch
 
 import pytest
@@ -34,19 +35,52 @@ def test_app_view_retrieve_panic(cache: Mock, client):
 
 
 @pytest.mark.django_db
-@pytest.mark.parametrize("metric", MetricType)
-def test_app_view_metrics(client, metric: MetricType):
+@pytest.mark.parametrize("metric,default_name", [(MetricType.WSGI, "path"), (MetricType.CELERY, "task")])
+def test_app_view_metrics(client, metric: MetricType, default_name: str):
     reference = "test"
     data = {"type": metric.value, "data": {"test": 1, "test1": 5}}
     url = reverse("sentry:apps-metrics", kwargs={"pk": reference, "metric_name": metric.value})
 
+    response = client.post(url, data, content_type="application/json")
+    assert response.status_code == 200, response.data
+    app = App.objects.get(reference=reference)
+    _, metric_data = app.get_metric(metric)
+    assert metric_data == {default_name: data["data"]}
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize("metric,default_name", [(MetricType.WSGI, "path"), (MetricType.CELERY, "task")])
+def test_app_view_metrics_new(client, metric: MetricType, default_name: str):
+    reference = "test"
+    data = {"type": metric.value, "data": {default_name: {"test": 1, "test1": 5}}}
+    url = reverse("sentry:apps-metrics", kwargs={"pk": reference, "metric_name": metric.value})
+
     response = client.post(url, data, content_type="application/json")
     assert response.status_code == 200, response.data
     app = App.objects.get(reference=reference)
     _, metric_data = app.get_metric(metric)
     assert metric_data == data["data"]
 
 
+@pytest.mark.django_db
+@pytest.mark.parametrize("metric,default_name", [(MetricType.WSGI, "path"), (MetricType.CELERY, "task")])
+def test_app_view_metrics_with_old_value(client, metric: MetricType, default_name: str):
+    reference = "test"
+    metrics = {"test1": 5, "test": 5}
+    app = App(reference=reference)
+    app.celery_metrics = metrics
+    app.wsgi_metrics = metrics
+    app.save()
+    data = {"type": metric.value, "data": {default_name: metrics}}
+    url = reverse("sentry:apps-metrics", kwargs={"pk": reference, "metric_name": metric.value})
+
+    response = client.post(url, data, content_type="application/json")
+    assert response.status_code == 200, response.data
+    app = App.objects.get(reference=reference)
+    _, metric_data = app.get_metric(metric)
+    assert metric_data == {default_name: Counter(metrics) + Counter(metrics)}
+
+
 @pytest.mark.django_db
 def test_app_view_list(client):
     response = client.get("/sentry/apps/")

controller/sentry/utils.py

@@ -56,3 +56,10 @@ def __call__(cls, *args, **kwargs):
         if cls not in cls._instances:
             cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
         return cls._instances[cls]
+
+
+def depth(_dict: dict) -> int:
+    """Compute the depth of a dict."""
+    if isinstance(_dict, dict):
+        return 1 + (max(map(depth, _dict.values())) if _dict else 0)
+    return 0

controller/settings.py

@@ -248,6 +248,8 @@
 
 DEFAULT_WSGI_IGNORE_PATHS = os.getenv("DEFAULT_WSGI_IGNORE_PATHS", "/health,/healthz,/health/,/healthz/").split(",")
 
+DEFAULT_WSGI_IGNORE_USER_AGENT = os.getenv("DEFAULT_WSGI_IGNORE_USER_AGENT", "kube-probe/").split(",")
+
 
 DEFAULT_CELERY_IGNORE_TASKS = []
 
@@ -338,7 +340,7 @@
 
     ENVIRONMENT = os.getenv("ENV", "production")
     sentry_sdk.init(
-        dsn="https://167f729cbcd24c1984d1721b6c3bf53b@o1290673.ingest.sentry.io/4504617124888576",
+        dsn=SENTRY_DSN,
         environment=ENVIRONMENT,
     )