opentelemetry-docker-tests: add declarative config end-to-end tests

.changelog/5740.added

@@ -0,0 +1 @@
+`opentelemetry-docker-tests`: add declarative config end-to-end tests

.github/workflows/generate_workflows.py

@@ -1,4 +1,5 @@
 from collections import defaultdict
+from configparser import ConfigParser
 from pathlib import Path
 from re import compile as re_compile
 
@@ -122,16 +123,30 @@ def get_misc_job_datas(tox_envs: list) -> list:
     return [tox_env for tox_env in tox_envs if not any(pattern.match(tox_env) for pattern in regex_patterns)]
 
 
+def get_contrib_job_datas(tox_ini_path: Path, job_datas: list) -> list:
+    """Return the jobs whose tox env installs packages from a contrib repo checkout."""
+    tox_ini = ConfigParser(interpolation=None, strict=False)
+    tox_ini.read(tox_ini_path)
+
+    return [
+        job_data
+        for job_data in job_datas
+        if "{env:CONTRIB_REPO_" in tox_ini.get(f"testenv:{job_data}", "deps", fallback="")
+    ]
+
+
 def _generate_workflow(
     job_datas: list,
     template_name: str,
     output_dir: Path,
+    **template_kwargs,
 ) -> None:
     env = Environment(loader=FileSystemLoader(Path(__file__).parent.joinpath("templates")))
     with open(output_dir.joinpath(f"{template_name}.yml"), "w") as yml_file:
         yml_file.write(
             env.get_template(f"{template_name}.yml.j2").render(
                 job_datas=job_datas,
+                **template_kwargs,
             )
         )
         yml_file.write("\n")
@@ -160,10 +175,12 @@ def generate_misc_workflow(
     tox_ini_path: Path,
     workflow_directory_path: Path,
 ) -> None:
+    misc_job_datas = get_misc_job_datas(get_tox_envs(tox_ini_path))
     _generate_workflow(
-        get_misc_job_datas(get_tox_envs(tox_ini_path)),
+        misc_job_datas,
         "misc",
         workflow_directory_path,
+        contrib_job_datas=get_contrib_job_datas(tox_ini_path, misc_job_datas),
     )
 
 

.github/workflows/misc.yml

@@ -23,6 +23,7 @@ env:
   PIP_EXISTS_ACTION: w
   CONTRIB_REPO_UTIL_HTTP: ${{ github.workspace }}/opentelemetry-python-contrib/util/opentelemetry-util-http
   CONTRIB_REPO_INSTRUMENTATION: ${{ github.workspace }}/opentelemetry-python-contrib/opentelemetry-instrumentation
+  CONTRIB_REPO_DISTRO: ${{ github.workspace }}/opentelemetry-python-contrib/opentelemetry-distro
   CONTRIB_REPO_INSTRUMENTATION_REQUESTS: ${{ github.workspace }}/opentelemetry-python-contrib/instrumentation/opentelemetry-instrumentation-requests
   CONTRIB_REPO_INSTRUMENTATION_WSGI: ${{ github.workspace }}/opentelemetry-python-contrib/instrumentation/opentelemetry-instrumentation-wsgi
   CONTRIB_REPO_INSTRUMENTATION_FLASK: ${{ github.workspace }}/opentelemetry-python-contrib/instrumentation/opentelemetry-instrumentation-flask
@@ -216,6 +217,35 @@ jobs:
       - name: Run tests
         run: uvx --with tox-uv tox -e docker-tests-opencensus
 
+  docker-tests-declconfig:
+    name: docker-tests-declconfig
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repo @ SHA - ${{ github.sha }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Checkout contrib repo @ ${{ env.CONTRIB_REPO_SHA }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          repository: open-telemetry/opentelemetry-python-contrib
+          ref: ${{ env.CONTRIB_REPO_SHA }}
+          path: opentelemetry-python-contrib
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.11"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+
+      - name: Run tests
+        run: uvx --with tox-uv tox -e docker-tests-declconfig
+
   public-symbols-check:
     name: public-symbols-check
     runs-on: ubuntu-latest

.github/workflows/templates/misc.yml.j2

@@ -23,6 +23,7 @@ env:
   PIP_EXISTS_ACTION: w
   CONTRIB_REPO_UTIL_HTTP: ${% raw %}{{ github.workspace }}{% endraw %}/opentelemetry-python-contrib/util/opentelemetry-util-http
   CONTRIB_REPO_INSTRUMENTATION: ${% raw %}{{ github.workspace }}{% endraw %}/opentelemetry-python-contrib/opentelemetry-instrumentation
+  CONTRIB_REPO_DISTRO: ${% raw %}{{ github.workspace }}{% endraw %}/opentelemetry-python-contrib/opentelemetry-distro
   CONTRIB_REPO_INSTRUMENTATION_REQUESTS: ${% raw %}{{ github.workspace }}{% endraw %}/opentelemetry-python-contrib/instrumentation/opentelemetry-instrumentation-requests
   CONTRIB_REPO_INSTRUMENTATION_WSGI: ${% raw %}{{ github.workspace }}{% endraw %}/opentelemetry-python-contrib/instrumentation/opentelemetry-instrumentation-wsgi
   CONTRIB_REPO_INSTRUMENTATION_FLASK: ${% raw %}{{ github.workspace }}{% endraw %}/opentelemetry-python-contrib/instrumentation/opentelemetry-instrumentation-flask
@@ -56,7 +57,7 @@ jobs:
         with:
           persist-credentials: false{% if job_data == "public-symbols-check" %}
           fetch-depth: 0{% endif %}
-      {%- if job_data == "tracecontext" %}
+      {%- if job_data in contrib_job_datas %}
 
       - name: Checkout contrib repo @ ${% raw %}{{ env.CONTRIB_REPO_SHA }}{% endraw %}
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

dev-requirements.txt

@@ -20,3 +20,4 @@ psutil==7.2.2
 GitPython==3.1.59
 pre-commit==3.7.0
 ruff==0.16.1
+testcontainers==4.13.3

tests/opentelemetry-docker-tests/tests/declconfig/README.md

@@ -0,0 +1,86 @@
+# Declarative configuration end-to-end tests
+
+Each scenario runs an app under `opentelemetry-instrument`, configured only by
+a declarative config file. The app exports to a real OpenTelemetry Collector,
+which forwards everything to an OTLP receiver in the test process (the sink).
+
+```mermaid
+flowchart LR
+    config[otel-config.yaml] -. configures .-> app["run.py<br/>(opentelemetry-instrument)"]
+    app -- OTLP --> collector["Collector<br/>(Docker)"]
+    collector -- otlphttp --> sink["Sink<br/>(in the test process)"]
+    sink --> checks["pytest<br/>until / expect"]
+```
+
+Run the suite with:
+
+```sh
+uv run tox -e docker-tests-declconfig
+```
+
+## Scenarios
+
+Every directory under `scenarios/` is one test case:
+
+| File                    | Purpose                                                                     |
+| ----------------------- | --------------------------------------------------------------------------- |
+| `scenario.yaml`         | Description, when to stop (`until`) and what to check (`expect`).           |
+| `otel-config.yaml`      | The SDK config, passed as `OTEL_CONFIG_FILE`. Can be `otel-config.json`.    |
+| `run.py`                | The app. Uses only the OpenTelemetry API.                                   |
+| `collector-config.yaml` | Optional. Defaults to the shared [`collector-config.yaml`](collector-config.yaml). |
+
+The shared collector receives OTLP on gRPC `4317` and HTTP `4318`. Configs
+reach it at `<scheme>://${OTEL_COLLECTOR_ENDPOINT}:<port>`. A custom collector
+config must enable `health_check` on `0.0.0.0:13133` and export with
+`otlphttp` to `${env:OTLP_SINK_ENDPOINT}`.
+
+The app doesn't inherit `OTEL_*` variables from the test environment.
+
+## `scenario.yaml`
+
+The full format is in [`scenario.schema.json`](scenario.schema.json).
+
+```yaml
+description: Exports a span over OTLP/HTTP.
+timeout: 30                     # seconds, default 30
+env: {MY_VAR: value}            # extra environment variables for the app
+until:                          # all must hold
+  - spans: {name: my-span, min_count: 1}
+  - metric: {name: requests, field: value, min: 5}   # field: value | count | sum
+  - logs: {body_pattern: "user .* in"}
+  - exited: {}                  # the app exited cleanly
+expect:
+  - resource:                   # every received item must match
+      schema_url: https://opentelemetry.io/schemas/1.26.0
+      attributes: {values: {service.name: my-service}}
+  - spans:
+      name: my-span
+      kind: INTERNAL
+      parent: {name: outer}     # nested span matcher
+      attributes: {values: {http.method: GET}, strict: true}
+      count: 1                  # or min_count (default 1) / max_count
+  - metric:
+      name: requests
+      type: sum
+      temporality: CUMULATIVE
+      data_point: {value: {gte: 3}, exemplar_count: {gte: 1}}
+  - logs:
+      body: {pattern: "user .* in"}
+      severity_number: WARN
+      span: {name: handler}
+  - output:
+      stream: stderr            # stdout | stderr
+      pattern: Unsupported file_format
+      exists: true              # false asserts no match
+```
+
+`count: 0` asserts that nothing matches. Pair it with `until: [exited: {}]`
+when nothing should be exported at all.
+
+Values are type strict: `5` matches only an int, `5.0` only a double and
+`"5"` only a string. Use `{double: 5}` or `{kvlist: {...}}` to set a type
+explicitly. Instead of a literal, a matcher can use the operators `pattern`,
+`exists`, `gt`, `gte`, `lt` and `lte`.
+
+A failure lists each unmet entry with its closest candidates, plus the app's
+stdout and stderr.