feat(aap): blocking id (#15042)

## Description

- new blocking id feature
- libddwaf update (required for this feature)
- refactor of blocking configuration with dedicated type
- tests updated
- will also be validated by system tests
DataDog/system-tests#5674

APPSEC-59798

Author: christophe-papazian

.github/workflows/system-tests.yml

@@ -45,7 +45,7 @@ jobs:
           persist-credentials: false
           repository: 'DataDog/system-tests'
           # Automatically managed, use scripts/update-system-tests-version to update
-          ref: '12860dc3fa4a474907f18f8313518989c80772ce'
+          ref: '279a4f17c9392157cdc106e627c2b57c2233899b'
 
       - name: Download wheels to binaries directory
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -90,7 +90,7 @@ jobs:
           persist-credentials: false
           repository: 'DataDog/system-tests'
           # Automatically managed, use scripts/update-system-tests-version to update
-          ref: '12860dc3fa4a474907f18f8313518989c80772ce'
+          ref: '279a4f17c9392157cdc106e627c2b57c2233899b'
 
       - name: Build runner
         uses: ./.github/actions/install_runner
@@ -275,7 +275,7 @@ jobs:
           persist-credentials: false
           repository: 'DataDog/system-tests'
           # Automatically managed, use scripts/update-system-tests-version to update
-          ref: '12860dc3fa4a474907f18f8313518989c80772ce'
+          ref: '279a4f17c9392157cdc106e627c2b57c2233899b'
       - name: Download wheels to binaries directory
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:

.gitlab-ci.yml

@@ -14,7 +14,7 @@ variables:
   DD_VPA_TEMPLATE: "vpa-template-cpu-p70-10percent-2x-oom-min-cap"
   # CI_DEBUG_SERVICES: "true"
   # Automatically managed, use scripts/update-system-tests-version to update
-  SYSTEM_TESTS_REF: "12860dc3fa4a474907f18f8313518989c80772ce"
+  SYSTEM_TESTS_REF: "279a4f17c9392157cdc106e627c2b57c2233899b"
 
 default:
   interruptible: true

ddtrace/appsec/_asm_request_context.py

@@ -18,6 +18,7 @@
 from ddtrace.appsec._constants import APPSEC
 from ddtrace.appsec._constants import SPAN_DATA_NAMES
 from ddtrace.appsec._constants import Constant_Class
+from ddtrace.appsec._utils import Block_config
 from ddtrace.appsec._utils import Telemetry_result
 from ddtrace.appsec._utils import get_triggers
 from ddtrace.contrib.internal.trace_utils_base import _normalize_tag_name
@@ -104,7 +105,7 @@ def __init__(self, span: Optional[Span] = None, rc_products: str = ""):
         self.telemetry: Telemetry_result = Telemetry_result()
         self.addresses_sent: Set[str] = set()
         self.waf_triggers: List[Dict[str, Any]] = []
-        self.blocked: Optional[Dict[str, Any]] = None
+        self.blocked: Optional[Block_config] = None
         self.finalized: bool = False
         self.api_security_reported: int = 0
         self.rc_products: str = rc_products
@@ -126,11 +127,11 @@ def is_blocked() -> bool:
     return env.blocked is not None
 
 
-def get_blocked() -> Dict[str, Any]:
+def get_blocked() -> Optional[Block_config]:
     env = _get_asm_context()
     if env is None:
-        return {}
-    return env.blocked or {}
+        return None
+    return env.blocked or None
 
 
 def get_entry_span() -> Optional[Span]:
@@ -200,15 +201,11 @@ def _use_html(headers) -> bool:
 
 def _ctype_from_headers(block_config, headers) -> None:
     """compute MIME type of the blocked response and store it in the block config"""
-    desired_type = block_config.get("type", "auto")
-    if desired_type == "auto":
-        block_config["content-type"] = "text/html" if _use_html(headers) else "application/json"
-    else:
-        block_config["content-type"] = "text/html" if block_config["type"] == "html" else "application/json"
+    if (block_config.type == "auto" and _use_html(headers)) or block_config.type == "html":
+        block_config.content_type = "text/html"
 
 
-def set_blocked(blocked: Dict[str, Any]) -> None:
-    blocked = blocked.copy()
+def set_blocked(blocked: Block_config) -> None:
     env = _get_asm_context()
     if env is None:
         logger.warning(WARNING_TAGS.SET_BLOCKED_NO_ASM_CONTEXT, extra=log_extra, stack_info=True)
@@ -217,6 +214,16 @@ def set_blocked(blocked: Dict[str, Any]) -> None:
     env.blocked = blocked
 
 
+def set_blocked_dict(block: Union[Dict[str, Any], Block_config, None]) -> None:
+    if isinstance(block, dict):
+        blocked = Block_config(**block)
+    elif block is None:
+        blocked = Block_config()
+    else:
+        blocked = block
+    set_blocked(blocked)
+
+
 def update_span_metrics(span: Span, name: str, value: Union[float, int]) -> None:
     span.set_metric(name, value + (span.get_metric(name) or 0.0))
 
@@ -713,7 +720,7 @@ def asm_listen():
     core.on("asgi.start_response", _call_waf)
     core.on("asgi.finalize_response", _set_headers_and_response)
 
-    core.on("asm.set_blocked", set_blocked)
+    core.on("asm.set_blocked", set_blocked_dict)
     core.on("asm.get_blocked", get_blocked, "block_config")
 
     core.on("context.ended.wsgi.__call__", _on_context_ended)

ddtrace/appsec/_constants.py

@@ -125,6 +125,7 @@ class APPSEC(metaclass=Constant_Class):
     ERROR_MESSAGE: Literal["_dd.appsec.error.message"] = "_dd.appsec.error.message"
     UNSUPPORTED_EVENT_TYPE: Literal["_dd.appsec.unsupported_event_type"] = "_dd.appsec.unsupported_event_type"
     SERVERLESS_TRACER_ENABLED: Literal["_dd.appsec.serverless.tracer"] = "_dd.appsec.serverless.tracer"
+    SECURITY_RESPONSE_ID: Literal["[security_response_id]"] = "[security_response_id]"
 
 
 TELEMETRY_OFF_NAME = "OFF"

ddtrace/appsec/_handlers.py

@@ -2,7 +2,9 @@
 import json
 from typing import Any
 from typing import Dict
+from typing import List
 from typing import Optional
+from typing import Tuple
 from typing import Union
 
 from ddtrace._trace.span import Span
@@ -15,6 +17,7 @@
 from ddtrace.appsec._http_utils import extract_cookies_from_headers
 from ddtrace.appsec._http_utils import normalize_headers
 from ddtrace.appsec._http_utils import parse_http_body
+from ddtrace.appsec._utils import Block_config
 from ddtrace.contrib import trace_utils
 from ddtrace.contrib.internal.trace_utils_base import _get_request_header_user_agent
 from ddtrace.contrib.internal.trace_utils_base import _set_url_tag
@@ -292,24 +295,23 @@ def _on_grpc_server_data(headers, request_message, method, metadata):
         set_waf_address(SPAN_DATA_NAMES.GRPC_SERVER_REQUEST_METADATA, dict(metadata))
 
 
-def _wsgi_make_block_content(ctx, construct_url):
+def _wsgi_make_block_content(ctx, construct_url) -> Tuple[int, List[Tuple[str, str]], bytes]:
     middleware = ctx.get_item("middleware")
     req_span = ctx.get_item("req_span")
     headers = ctx.get_item("headers")
     environ = ctx.get_item("environ")
     if req_span is None:
         raise ValueError("request span not found")
-    block_config = get_blocked()
-    desired_type = block_config.get("type", "auto")
+    block_config: Block_config = get_blocked() or Block_config()
     ctype = None
-    if desired_type == "none":
-        content = ""
-        resp_headers = [("content-type", "text/plain; charset=utf-8"), ("location", block_config.get("location", ""))]
+    if block_config.type == "none":
+        content = b""
+        resp_headers = [("content-type", "text/plain; charset=utf-8"), ("location", block_config.location)]
     else:
-        ctype = block_config.get("content-type", "application/json")
-        content = http_utils._get_blocked_template(ctype).encode("UTF-8")
+        ctype = block_config.content_type
+        content = http_utils._get_blocked_template(ctype, block_config.block_id).encode("UTF-8")
         resp_headers = [("content-type", ctype)]
-    status = block_config.get("status_code", 403)
+    status = block_config.status_code
     try:
         req_span._set_tag_str(RESPONSE_HEADERS + ".content-length", str(len(content)))
         if ctype is not None:
@@ -332,28 +334,26 @@ def _wsgi_make_block_content(ctx, construct_url):
     return status, resp_headers, content
 
 
-def _asgi_make_block_content(ctx, url):
+def _asgi_make_block_content(ctx, url) -> Tuple[int, List[Tuple[bytes, bytes]], bytes]:
     middleware = ctx.get_item("middleware")
     req_span = ctx.get_item("req_span")
     headers = ctx.get_item("headers")
     environ = ctx.get_item("environ")
     if req_span is None:
         raise ValueError("request span not found")
-    block_config = get_blocked()
-    desired_type = block_config.get("type", "auto")
+    block_config = get_blocked() or Block_config()
     ctype = None
-    if desired_type == "none":
-        content = ""
+    if block_config.type == "none":
+        content = b""
         resp_headers = [
             (b"content-type", b"text/plain; charset=utf-8"),
-            (b"location", block_config.get("location", "").encode()),
+            (b"location", block_config.location.encode()),
         ]
     else:
-        ctype = block_config.get("content-type", "application/json")
-        content = http_utils._get_blocked_template(ctype).encode("UTF-8")
+        content = http_utils._get_blocked_template(block_config.content_type, block_config.block_id).encode("UTF-8")
         # ctype = f"{ctype}; charset=utf-8" can be considered at some point
-        resp_headers = [(b"content-type", ctype.encode())]
-    status = block_config.get("status_code", 403)
+        resp_headers = [(b"content-type", block_config.content_type.encode())]
+    status = block_config.status_code
     try:
         req_span._set_tag_str(RESPONSE_HEADERS + ".content-length", str(len(content)))
         if ctype is not None:

ddtrace/appsec/_processor.py

@@ -35,6 +35,7 @@
 from ddtrace.appsec._exploit_prevention.stack_traces import report_stack
 from ddtrace.appsec._trace_utils import _asm_manual_keep
 from ddtrace.appsec._utils import Binding_error
+from ddtrace.appsec._utils import Block_config
 from ddtrace.appsec._utils import DDWaf_result
 from ddtrace.constants import _ORIGIN_KEY
 from ddtrace.constants import _RUNTIME_FAMILY
@@ -341,7 +342,7 @@ def _waf_action(
             log.debug("[DDAS-011-00] ASM In-App WAF returned: %s. Timeout %s", waf_results.data, waf_results.timeout)
 
         if blocked:
-            _asm_request_context.set_blocked(blocked)
+            _asm_request_context.set_blocked(Block_config(**blocked))
 
         allowed = True
         if waf_results.keep:

ddtrace/appsec/_utils.py

@@ -154,6 +154,26 @@ def __init__(self) -> None:
         self.durations: Dict[str, float] = collections.defaultdict(float)
 
 
+class Block_config:
+    __slots__ = ["block_id", "grpc_status_code", "status_code", "type", "content_type", "location"]
+
+    def __init__(
+        self,
+        type: str = "auto",  # noqa: A002
+        status_code: int = 403,
+        grpc_status_code: int = 10,
+        security_response_id: str = "default",
+        location: str = "",
+        **_kwargs,
+    ) -> None:
+        self.block_id: str = security_response_id
+        self.grpc_status_code: int = grpc_status_code
+        self.status_code: int = status_code
+        self.type: str = type
+        self.location = location.replace(APPSEC.SECURITY_RESPONSE_ID, security_response_id)
+        self.content_type: str = "application/json"
+
+
 class Telemetry_result:
     __slots__ = [
         "blocked",

ddtrace/contrib/internal/django/response.py

@@ -29,6 +29,7 @@
 from ddtrace.internal.logger import get_logger
 from ddtrace.internal.schema import schematize_url_operation
 from ddtrace.internal.schema.span_attribute_schema import SpanDirection
+from ddtrace.internal.utils import Block_config
 from ddtrace.internal.utils import get_argument_value
 from ddtrace.internal.utils import get_blocked
 from ddtrace.internal.utils import http as http_utils
@@ -119,27 +120,24 @@ def traced_get_response(func: FunctionType, args: Tuple[Any, ...], kwargs: Dict[
 
         response = None
 
-        def blocked_response():
-            block_config = get_blocked() or {}
-            desired_type = block_config.get("type", "auto")
-            status = block_config.get("status_code", 403)
-            if desired_type == "none":
-                response = HttpResponse("", status=status)
-                location = block_config.get("location", "")
-                if location:
-                    response["location"] = location
+        def blocked_response(block_config: Block_config):
+            if block_config.type == "none":
+                response = HttpResponse("", status=block_config.status_code)
+                if block_config.location:
+                    response["location"] = block_config.location
             else:
-                ctype = block_config.get("content-type", "application/json")
-                content = http_utils._get_blocked_template(ctype)
-                response = HttpResponse(content, content_type=ctype, status=status)
+                content = http_utils._get_blocked_template(block_config.content_type, block_config.block_id)
+                response = HttpResponse(
+                    content, content_type=block_config.content_type, status=block_config.status_code
+                )
                 response.content = content
                 response["Content-Length"] = len(content.encode())
             utils._after_request_tags(pin, ctx.span, request, response)
             return response
 
         try:
-            if get_blocked():
-                response = blocked_response()
+            if block_config := get_blocked():
+                response = blocked_response(block_config)
             else:
                 query = request.META.get("QUERY_STRING", "")
                 uri = utils.get_request_uri(request)
@@ -158,25 +156,25 @@ def blocked_response():
                 )
                 core.dispatch("django.start_response.post", ("Django",))
 
-                if get_blocked():
-                    response = blocked_response()
+                if block_config := get_blocked():
+                    response = blocked_response(block_config)
                 else:
                     try:
                         response = func(*args, **kwargs)
                     except BlockingException as e:
                         set_blocked(e.args[0])
-                        response = blocked_response()
+                        response = blocked_response(e.args[0])
                         return response
 
-                    if get_blocked():
-                        response = blocked_response()
+                    if block_config := get_blocked():
+                        response = blocked_response(block_config)
 
         finally:
             core.dispatch("django.finalize_response.pre", (ctx, utils._after_request_tags, request, response))
             if not get_blocked():
                 core.dispatch("django.finalize_response", ("Django",))
-                if get_blocked():
-                    response = blocked_response()
+                if block_config := get_blocked():
+                    response = blocked_response(block_config)
         return response
 
 

ddtrace/contrib/internal/flask/patch.py

@@ -125,7 +125,7 @@ def _wrapped_start_response(self, start_response, ctx, status_code, headers, exc
         core.dispatch("flask.start_response.pre", (flask.request, ctx, config.flask, status_code, headers))
         if not get_blocked():
             core.dispatch("flask.start_response", ("Flask",))
-            if get_blocked():
+            if block_config := get_blocked():
                 # response code must be set here, or it will be too late
                 result_content = core.dispatch_with_results(  # ast-grep-ignore: core-dispatch-with-results
                     "flask.block.request.content", ()
@@ -134,16 +134,13 @@ def _wrapped_start_response(self, start_response, ctx, status_code, headers, exc
                     _, status, response_headers = result_content.value
                     result = start_response(str(status), response_headers)
                 else:
-                    block_config = get_blocked()
-                    desired_type = block_config.get("type", "auto")
-                    status = block_config.get("status_code", 403)
-                    if desired_type == "none":
-                        response_headers = []
-                    else:
-                        ctype = block_config.get("content-type", "application/json")
-                        response_headers = [("content-type", ctype)]
-                    result = start_response(str(status), response_headers)
-                core.dispatch("flask.start_response.blocked", (ctx, config.flask, response_headers, status))
+                    response_headers = (
+                        [] if block_config.type == "none" else [("content-type", block_config.content_type)]
+                    )
+                    result = start_response(str(block_config.status_code), response_headers)
+                core.dispatch(
+                    "flask.start_response.blocked", (ctx, config.flask, response_headers, block_config.status_code)
+                )
             else:
                 result = start_response(status_code, headers)
         else:
@@ -549,8 +546,10 @@ def _wrap(code_or_exception, f):
 def _block_request_callable(call):
     set_blocked()
     core.dispatch("flask.blocked_request_callable", (call,))
-    ctype = get_blocked().get("content-type", "application/json")
-    abort(flask.Response(http_utils._get_blocked_template(ctype), content_type=ctype, status=403))
+    block_config = get_blocked()
+    ctype = block_config.content_type if block_config else "application/json"
+    block_id = block_config.block_id if block_config else "(default)"
+    abort(flask.Response(http_utils._get_blocked_template(ctype, block_id), content_type=ctype, status=403))
 
 
 def request_patcher(name):