From 8af809484a56adc91943a52d74b63b889e5fe67b Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 9 Jun 2026 11:28:26 -0700 Subject: [PATCH 1/9] fix: Workflow github --- .github/memory-benchmark.yml | 40 +++++++++ Makefile | 4 +- tests/perf/README.md | 10 +++ tests/perf/baseline.json | 158 +++++++++++++++++------------------ tests/perf/run_profile.py | 46 ++++++++++ 5 files changed, 178 insertions(+), 80 deletions(-) create mode 100644 .github/memory-benchmark.yml diff --git a/.github/memory-benchmark.yml b/.github/memory-benchmark.yml new file mode 100644 index 00000000..44d968f9 --- /dev/null +++ b/.github/memory-benchmark.yml @@ -0,0 +1,40 @@ +name: Memory benchmark + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + +permissions: + contents: read + +jobs: + memory-benchmark: + name: Memory benchmark + # Needs to match the arch the baseline was generated on. + runs-on: ubuntu-24.04-arm + if: | + contains(github.event.pull_request.labels.*.name, 'check-memory-benchmark') && + ( + github.event.pull_request.author_association == 'COLLABORATOR' || + github.event.pull_request.author_association == 'MEMBER' || + github.event.pull_request.author_association == 'OWNER' + ) + steps: + - uses: actions/checkout@v4 + + # Uses the Dockerfile environment for repeatable runs. + - name: Run memory benchmark + run: make memory-use-bench + + # Upload all three flamegraph views per scenario (peak/leaks/temporary). + - name: Upload flamegraph reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: memray-flamegraphs + path: tests/perf/reports/*.html + if-no-files-found: warn diff --git a/Makefile b/Makefile index 4070ff0e..4da1a66c 100644 --- a/Makefile +++ b/Makefile @@ -148,10 +148,12 @@ MEMRAY_ITERATIONS ?= 100 MEMRAY_THRESHOLD ?= 1.1 SCENARIO ?= SCENARIO_ARG := $(if $(SCENARIO),--scenario $(SCENARIO),) +# In CI, use en vars to write the report to the job run +GH_SUMMARY_MOUNT := $(if $(GITHUB_STEP_SUMMARY),-v $(GITHUB_STEP_SUMMARY):$(GITHUB_STEP_SUMMARY),) .PHONY: memory-use-bench memory-use-bench: docker build -f tests/perf/Dockerfiles/$(PERF_ENV)-perf-Dockerfile -t c2pa-memray-$(PERF_ENV) . - docker run --rm -v $(PWD):/workspace -e PYTHONPATH=/workspace/src -e PERF_ENV=$(PERF_ENV) -e MEMRAY_ITERATIONS=$(MEMRAY_ITERATIONS) -e MEMRAY_THRESHOLD=$(MEMRAY_THRESHOLD) c2pa-memray-$(PERF_ENV) python -m tests.perf.run_profile $(SCENARIO_ARG) $(PERF_ARGS) + docker run --rm -v $(PWD):/workspace $(GH_SUMMARY_MOUNT) -e PYTHONPATH=/workspace/src -e PERF_ENV=$(PERF_ENV) -e MEMRAY_ITERATIONS=$(MEMRAY_ITERATIONS) -e MEMRAY_THRESHOLD=$(MEMRAY_THRESHOLD) -e GITHUB_TOKEN -e GITHUB_STEP_SUMMARY c2pa-memray-$(PERF_ENV) python -m tests.perf.run_profile $(SCENARIO_ARG) $(PERF_ARGS) @echo "" @echo "Reports written to tests/perf/reports/" @echo "Open tests/perf/reports/-{peak,leaks,temporary}.html in a browser" diff --git a/tests/perf/README.md b/tests/perf/README.md index 1f2ec022..52b12bad 100644 --- a/tests/perf/README.md +++ b/tests/perf/README.md @@ -67,6 +67,16 @@ The trailing `VAR=value` arguments (e.g. `PERF_ENV=ubuntu-24.04`, `PERF_ARGS=--u Reports are written to `tests/perf/reports/` on the local machine. Three HTML files per scenario, one per suffix (described below). Open any in a browser. After a run, the run also reports if the scenarios were or were not all within baseline threshold (baseline +10% memory use tolerance). +## Running in CI + +The `.github/workflows/memory-benchmark.yml` workflow runs the Docker-based benchmarks on a PR, but only when the PR has the `check-memory-benchmark` label. This runs `make memory-use-bench`, so: + +- A regression (peak or leaked > baseline +10%) makes the benchmark job exit non-zero. +- A values report table is written to the job's Step Summary. +- All three flamegraph HTML views per scenario are uploaded as the `memray-flamegraphs` artifact. + +The gate only acts as regression test once a `tests/perf/baseline.json` is committed on the branch. Without one, `run_profile.py` treats the run as baseline creation (exits 0, no gating). + ## Report views Each scenario produces three [memray flamegraphs](https://bloomberg.github.io/memray/flamegraph.html). All three are flamegraphs of the same run. They differ only in which allocations they count. diff --git a/tests/perf/baseline.json b/tests/perf/baseline.json index 302d648a..bd4265f9 100644 --- a/tests/perf/baseline.json +++ b/tests/perf/baseline.json @@ -2,139 +2,139 @@ "_meta": { "memray_version": "1.19.3", "python_version": "3.12.13", - "c2pa_native_version": "c2pa-v0.85.1", + "c2pa_native_version": "c2pa-v0.86.1", "iterations": 100, "perf_env": "python-3.12-slim", "arch": "aarch64" }, "reader_jpeg_legacy": { - "peak_bytes": 3814421, - "leaked_bytes": 3266116, - "total_allocations": 698899 + "peak_bytes": 3790506, + "leaked_bytes": 3337275, + "total_allocations": 694247 }, "reader_jpeg_with_context": { - "peak_bytes": 3822953, - "leaked_bytes": 3257471, - "total_allocations": 692953 + "peak_bytes": 3784932, + "leaked_bytes": 3331065, + "total_allocations": 688204 }, "reader_mp4": { - "peak_bytes": 4876441, - "leaked_bytes": 3257485, - "total_allocations": 2112991 + "peak_bytes": 4192391, + "leaked_bytes": 3331202, + "total_allocations": 1856171 }, "reader_wav": { - "peak_bytes": 5520266, - "leaked_bytes": 3267427, - "total_allocations": 400371 + "peak_bytes": 4440733, + "leaked_bytes": 3281502, + "total_allocations": 386809 }, "builder_sign_jpeg_legacy": { - "peak_bytes": 7695310, - "leaked_bytes": 3383623, - "total_allocations": 522425 + "peak_bytes": 7739038, + "leaked_bytes": 3438581, + "total_allocations": 515197 }, "builder_sign_jpeg_with_context": { - "peak_bytes": 7688236, - "leaked_bytes": 3376293, - "total_allocations": 516851 + "peak_bytes": 7732082, + "leaked_bytes": 3431373, + "total_allocations": 509328 }, "builder_sign_png_legacy": { - "peak_bytes": 7932767, - "leaked_bytes": 3383648, - "total_allocations": 1694629 + "peak_bytes": 7939922, + "leaked_bytes": 3401346, + "total_allocations": 1399159 }, "builder_sign_png_with_context": { - "peak_bytes": 7925490, - "leaked_bytes": 3376452, - "total_allocations": 1688908 + "peak_bytes": 7932212, + "leaked_bytes": 3393558, + "total_allocations": 1393313 }, "builder_sign_jpeg_parallel_split_pool": { - "peak_bytes": 45764159, - "leaked_bytes": 3818113, - "total_allocations": 528785 + "peak_bytes": 45802848, + "leaked_bytes": 3855293, + "total_allocations": 520103 }, "builder_sign_jpeg_parallel_split_barrier": { - "peak_bytes": 46225287, - "leaked_bytes": 3809216, - "total_allocations": 527412 + "peak_bytes": 45771974, + "leaked_bytes": 3853543, + "total_allocations": 519696 }, "builder_sign_png_parallel_split_pool": { - "peak_bytes": 46002549, - "leaked_bytes": 3817801, - "total_allocations": 1700731 + "peak_bytes": 46511069, + "leaked_bytes": 3831120, + "total_allocations": 1405010 }, "builder_sign_png_parallel_split_barrier": { - "peak_bytes": 45970433, - "leaked_bytes": 3812044, - "total_allocations": 1699396 + "peak_bytes": 45980602, + "leaked_bytes": 3826012, + "total_allocations": 1403879 }, "builder_sign_gif": { - "peak_bytes": 14544515, - "leaked_bytes": 3375865, - "total_allocations": 7183237 + "peak_bytes": 14556418, + "leaked_bytes": 3398315, + "total_allocations": 5573733 }, "builder_sign_heic": { - "peak_bytes": 4608484, - "leaked_bytes": 3376030, - "total_allocations": 771079 + "peak_bytes": 4624733, + "leaked_bytes": 3402824, + "total_allocations": 733409 }, "builder_sign_m4a": { - "peak_bytes": 18849082, - "leaked_bytes": 3376431, - "total_allocations": 2273497 + "peak_bytes": 18864770, + "leaked_bytes": 3402727, + "total_allocations": 2126542 }, "builder_sign_webp": { - "peak_bytes": 8900701, - "leaked_bytes": 3376432, - "total_allocations": 487683 + "peak_bytes": 8911369, + "leaked_bytes": 3397659, + "total_allocations": 473589 }, "builder_sign_avi": { - "peak_bytes": 7040387, - "leaked_bytes": 3376267, - "total_allocations": 40315553 + "peak_bytes": 7357514, + "leaked_bytes": 3610304, + "total_allocations": 34863789 }, "builder_sign_mp4": { - "peak_bytes": 6162851, - "leaked_bytes": 3376431, - "total_allocations": 1809672 + "peak_bytes": 6202809, + "leaked_bytes": 3426948, + "total_allocations": 1595487 }, "builder_sign_tiff": { - "peak_bytes": 13124728, - "leaked_bytes": 3376268, - "total_allocations": 5139967 + "peak_bytes": 13430248, + "leaked_bytes": 3661058, + "total_allocations": 5484643 }, "builder_sign_jpeg_parent_of": { - "peak_bytes": 14173992, - "leaked_bytes": 3377656, - "total_allocations": 1209933 + "peak_bytes": 14244270, + "leaked_bytes": 3457332, + "total_allocations": 1194872 }, "builder_sign_jpeg_component_of": { - "peak_bytes": 14175518, - "leaked_bytes": 3377891, - "total_allocations": 1232336 + "peak_bytes": 14246032, + "leaked_bytes": 3457636, + "total_allocations": 1217308 }, "builder_sign_jpeg_parent_and_component": { - "peak_bytes": 14530406, - "leaked_bytes": 3474418, - "total_allocations": 2160934 + "peak_bytes": 14602066, + "leaked_bytes": 3560736, + "total_allocations": 2137378 }, "builder_sign_jpeg_parent_and_component_mixed_mime": { - "peak_bytes": 14476171, - "leaked_bytes": 3378735, - "total_allocations": 2451587 + "peak_bytes": 14553513, + "leaked_bytes": 3464495, + "total_allocations": 2148041 }, "builder_sign_jpeg_two_components_same_mime": { - "peak_bytes": 14519270, - "leaked_bytes": 3473673, - "total_allocations": 2150782 + "peak_bytes": 14582525, + "leaked_bytes": 3560177, + "total_allocations": 2127097 }, "builder_sign_jpeg_two_components_mixed_mime": { - "peak_bytes": 14473127, - "leaked_bytes": 3377445, - "total_allocations": 2441195 + "peak_bytes": 14549485, + "leaked_bytes": 3462954, + "total_allocations": 2137855 }, "builder_sign_jpeg_archive_roundtrip": { - "peak_bytes": 14226832, - "leaked_bytes": 3426491, - "total_allocations": 1680290 + "peak_bytes": 14327685, + "leaked_bytes": 3527329, + "total_allocations": 1657777 } } \ No newline at end of file diff --git a/tests/perf/run_profile.py b/tests/perf/run_profile.py index 31593967..df469b47 100644 --- a/tests/perf/run_profile.py +++ b/tests/perf/run_profile.py @@ -169,6 +169,49 @@ def _fmt(n: int) -> str: return f"{n} B" +def _delta_pct(current: int, base: int) -> str: + """Signed percentage change vs baseline, or '-' when no baseline.""" + if not base: + return "-" + return f"{(current - base) / base * 100:+.1f}%" + + +def _write_github_summary(results: dict, baseline: dict) -> None: + """Append a values table to $GITHUB_STEP_SUMMARY when running in CI. + """ + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path or not results: + return + + lines = [ + "## Memory benchmark (memray)", + "", + f"Iterations: {ITERATIONS} · threshold: +{(THRESHOLD - 1) * 100:.0f}%" + f"{f' · env: {PERF_ENV}' if PERF_ENV else ''}", + "", + "| scenario | peak | leaked | allocs | peak Δ% | leaked Δ% | status |", + "|----------|------|--------|--------|---------|-----------|--------|", + ] + for name, m in results.items(): + b = baseline.get(name, {}) if baseline else {} + peak_base = b.get("peak_bytes", 0) + leaked_base = b.get("leaked_bytes", 0) + regressed = ( + (peak_base and m["peak_bytes"] > peak_base * THRESHOLD) + or (leaked_base and m["leaked_bytes"] > leaked_base * THRESHOLD) + ) + status = "REGRESSED" if regressed else "ok" + lines.append( + f"| {name} | {_fmt(m['peak_bytes'])} | {_fmt(m['leaked_bytes'])} " + f"| {m['total_allocations']} | {_delta_pct(m['peak_bytes'], peak_base)} " + f"| {_delta_pct(m['leaked_bytes'], leaked_base)} | {status} |" + ) + lines.append("") + + with open(summary_path, "a", encoding="utf-8") as fh: + fh.write("\n".join(lines) + "\n") + + def main() -> None: parser = argparse.ArgumentParser(description="c2pa-python memory profiler") parser.add_argument( @@ -274,6 +317,9 @@ def main() -> None: verb = "Updated" if baseline else "Created" print(f"\n{verb} baseline: {BASELINE_FILE}") + # Emit the report table to the PR's Step Summary in CI. + _write_github_summary(results, baseline) + if render_failures: print("\nFLAMEGRAPH RENDERS FAILED (capture + metrics still recorded):", file=sys.stderr) for r in render_failures: From d2277f035088136f2eb65fbeb510d05ef32cadba Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 9 Jun 2026 11:32:18 -0700 Subject: [PATCH 2/9] fix: Put workflow file in proper lcoation --- .github/{ => workflows}/memory-benchmark.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{ => workflows}/memory-benchmark.yml (100%) diff --git a/.github/memory-benchmark.yml b/.github/workflows/memory-benchmark.yml similarity index 100% rename from .github/memory-benchmark.yml rename to .github/workflows/memory-benchmark.yml From b0099b2a35eca9eb77fc0ba2a425f823ff107728 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 9 Jun 2026 12:40:37 -0700 Subject: [PATCH 3/9] fix: Workflow name --- .github/workflows/memory-benchmark.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/memory-benchmark.yml b/.github/workflows/memory-benchmark.yml index 44d968f9..5232e1ee 100644 --- a/.github/workflows/memory-benchmark.yml +++ b/.github/workflows/memory-benchmark.yml @@ -1,4 +1,4 @@ -name: Memory benchmark +name: Python SDK memray memory benchmark on: pull_request: @@ -13,7 +13,7 @@ permissions: jobs: memory-benchmark: - name: Memory benchmark + name: Python SDK memray memory benchmark # Needs to match the arch the baseline was generated on. runs-on: ubuntu-24.04-arm if: | @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@v4 # Uses the Dockerfile environment for repeatable runs. - - name: Run memory benchmark + - name: Run memray memory benchmark run: make memory-use-bench # Upload all three flamegraph views per scenario (peak/leaks/temporary). From 3bc250f17c5173743a7062daddeaa1ff13abfaa8 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 30 Jun 2026 20:12:14 -0700 Subject: [PATCH 4/9] fix: Update baseline --- .../python-3.10-slim-perf-Dockerfile | 7 +- .../python-3.12-slim-perf-Dockerfile | 7 +- tests/perf/baseline.json | 260 ++++++++++-------- tests/perf/run_profile.py | 6 +- 4 files changed, 160 insertions(+), 120 deletions(-) diff --git a/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile b/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile index 0db28b11..b6f37f90 100644 --- a/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile +++ b/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile @@ -2,9 +2,14 @@ FROM python:3.10.20-slim-bookworm WORKDIR /workspace -# libunwind for memray native stack unwinding +# libunwind for memray native stack unwinding. +# python3-dbg + libc6-dbg supply the debug symbols memray needs to resolve +# native (C) frames; without them --native flamegraphs warn "No debug +# information was found for the Python interpreter" and show unnamed frames. RUN apt-get update && apt-get install -y --no-install-recommends \ libunwind-dev \ + python3-dbg \ + libc6-dbg \ ca-certificates \ && rm -rf /var/lib/apt/lists/* diff --git a/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile b/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile index 1e387d1c..e98c0fd4 100644 --- a/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile +++ b/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile @@ -2,9 +2,14 @@ FROM python:3.12.13-slim-bookworm WORKDIR /workspace -# libunwind for memray native stack unwinding +# libunwind for memray native stack unwinding. +# python3-dbg + libc6-dbg supply the debug symbols memray needs to resolve +# native (C) frames; without them --native flamegraphs warn "No debug +# information was found for the Python interpreter" and show unnamed frames. RUN apt-get update && apt-get install -y --no-install-recommends \ libunwind-dev \ + python3-dbg \ + libc6-dbg \ ca-certificates \ && rm -rf /var/lib/apt/lists/* diff --git a/tests/perf/baseline.json b/tests/perf/baseline.json index 6908e7da..7af9ffb5 100644 --- a/tests/perf/baseline.json +++ b/tests/perf/baseline.json @@ -2,194 +2,224 @@ "_meta": { "memray_version": "1.19.3", "python_version": "3.12.13", - "c2pa_native_version": "c2pa-v0.88.0", - "iterations": 250, + "c2pa_native_version": "c2pa-v0.89.0", + "iterations": 100, "perf_env": "python-3.12-slim", "arch": "aarch64" }, "reader_jpeg_legacy": { - "peak_bytes": 3762136, - "leaked_bytes": 3262031, - "total_allocations": 1660100 + "peak_bytes": 3791301, + "leaked_bytes": 3292672, + "total_allocations": 722823 }, "reader_jpeg_with_context": { - "peak_bytes": 3756118, - "leaked_bytes": 3254069, - "total_allocations": 1646550 + "peak_bytes": 3785583, + "leaked_bytes": 3284873, + "total_allocations": 717272 + }, + "reader_manifest_data_context": { + "peak_bytes": 7576945, + "leaked_bytes": 3407648, + "total_allocations": 619356 }, "reader_mp4": { - "peak_bytes": 4208976, - "leaked_bytes": 3253426, - "total_allocations": 4963836 + "peak_bytes": 4159582, + "leaked_bytes": 3283805, + "total_allocations": 2089508 }, "reader_wav": { - "peak_bytes": 4431268, - "leaked_bytes": 3263354, - "total_allocations": 888612 + "peak_bytes": 4464615, + "leaked_bytes": 3293702, + "total_allocations": 413484 }, "builder_sign_jpeg_legacy": { - "peak_bytes": 7694334, - "leaked_bytes": 3377383, - "total_allocations": 1250156 + "peak_bytes": 7724599, + "leaked_bytes": 3408513, + "total_allocations": 560691 }, "builder_sign_jpeg_with_context": { - "peak_bytes": 7687683, - "leaked_bytes": 3370546, - "total_allocations": 1235555 + "peak_bytes": 7716613, + "leaked_bytes": 3400514, + "total_allocations": 554788 }, "builder_sign_png_legacy": { - "peak_bytes": 7933540, - "leaked_bytes": 3377209, - "total_allocations": 4502325 + "peak_bytes": 7961139, + "leaked_bytes": 3406984, + "total_allocations": 1981843 }, "builder_sign_png_with_context": { - "peak_bytes": 7924904, - "leaked_bytes": 3370312, - "total_allocations": 4487743 + "peak_bytes": 7955799, + "leaked_bytes": 3401929, + "total_allocations": 1975867 }, "builder_sign_jpeg_parallel_split_pool": { - "peak_bytes": 45790186, - "leaked_bytes": 3770876, - "total_allocations": 1247400 + "peak_bytes": 44002804, + "leaked_bytes": 3801899, + "total_allocations": 566497 }, "builder_sign_jpeg_parallel_split_barrier": { - "peak_bytes": 45758269, - "leaked_bytes": 3769617, - "total_allocations": 1246063 + "peak_bytes": 45784452, + "leaked_bytes": 3800550, + "total_allocations": 565361 }, "builder_sign_png_parallel_split_pool": { - "peak_bytes": 42884934, - "leaked_bytes": 3806845, - "total_allocations": 4499456 + "peak_bytes": 46054163, + "leaked_bytes": 3801867, + "total_allocations": 1987868 }, "builder_sign_png_parallel_split_barrier": { - "peak_bytes": 45995848, - "leaked_bytes": 3805431, - "total_allocations": 4498202 + "peak_bytes": 44206900, + "leaked_bytes": 3800490, + "total_allocations": 1986526 }, "builder_sign_gif": { - "peak_bytes": 14545704, - "leaked_bytes": 3369959, - "total_allocations": 19592169 + "peak_bytes": 14574556, + "leaked_bytes": 3400735, + "total_allocations": 8550061 }, "builder_sign_heic": { - "peak_bytes": 4609675, - "leaked_bytes": 3369960, - "total_allocations": 1865279 + "peak_bytes": 4644811, + "leaked_bytes": 3407275, + "total_allocations": 831824 }, "builder_sign_m4a": { - "peak_bytes": 18848657, - "leaked_bytes": 3369911, - "total_allocations": 6143845 + "peak_bytes": 18885052, + "leaked_bytes": 3407378, + "total_allocations": 2647270 }, "builder_sign_webp": { - "peak_bytes": 8901476, - "leaked_bytes": 3369960, - "total_allocations": 1108971 + "peak_bytes": 8928848, + "leaked_bytes": 3399564, + "total_allocations": 499272 }, "builder_sign_avi": { - "peak_bytes": 7041162, - "leaked_bytes": 3369959, - "total_allocations": 105383089 + "peak_bytes": 7068483, + "leaked_bytes": 3399340, + "total_allocations": 45032279 }, "builder_sign_mp4": { - "peak_bytes": 6163626, - "leaked_bytes": 3369959, - "total_allocations": 4502723 + "peak_bytes": 6198764, + "leaked_bytes": 3407319, + "total_allocations": 1944326 }, "builder_sign_tiff": { - "peak_bytes": 13123408, - "leaked_bytes": 3369960, - "total_allocations": 13221796 + "peak_bytes": 13151779, + "leaked_bytes": 3400355, + "total_allocations": 5472611 }, "builder_sign_jpeg_parent_of": { - "peak_bytes": 14175499, - "leaked_bytes": 3370412, - "total_allocations": 3049271 + "peak_bytes": 14204315, + "leaked_bytes": 3401481, + "total_allocations": 1292637 }, "builder_sign_jpeg_component_of": { - "peak_bytes": 14176177, - "leaked_bytes": 3370939, - "total_allocations": 3105680 + "peak_bytes": 14204923, + "leaked_bytes": 3400730, + "total_allocations": 1315125 }, "builder_sign_jpeg_parent_and_component": { - "peak_bytes": 14521625, - "leaked_bytes": 3466959, - "total_allocations": 5528187 + "peak_bytes": 14588185, + "leaked_bytes": 3549911, + "total_allocations": 2299453 }, "builder_sign_jpeg_parent_and_component_mixed_mime": { - "peak_bytes": 14476222, - "leaked_bytes": 3370421, - "total_allocations": 6500129 + "peak_bytes": 14506586, + "leaked_bytes": 3401364, + "total_allocations": 2796028 }, "builder_sign_jpeg_two_components_same_mime": { - "peak_bytes": 14584523, - "leaked_bytes": 3506901, - "total_allocations": 5502714 + "peak_bytes": 14601941, + "leaked_bytes": 3558116, + "total_allocations": 2289245 }, "builder_sign_jpeg_two_components_mixed_mime": { - "peak_bytes": 14473585, - "leaked_bytes": 3370669, - "total_allocations": 6474343 + "peak_bytes": 14503695, + "leaked_bytes": 3401276, + "total_allocations": 2785702 }, "builder_sign_jpeg_archive_roundtrip": { - "peak_bytes": 14206327, - "leaked_bytes": 3389587, - "total_allocations": 4247160 + "peak_bytes": 14236247, + "leaked_bytes": 3420354, + "total_allocations": 1777705 + }, + "builder_to_archive_with_ingredient": { + "peak_bytes": 14010137, + "leaked_bytes": 3278101, + "total_allocations": 957452 + }, + "builder_sign_jpeg_archive_roundtrip_ingredient_in_archive": { + "peak_bytes": 14225579, + "leaked_bytes": 3420822, + "total_allocations": 2981495 + }, + "builder_write_ingredient_archive": { + "peak_bytes": 14010131, + "leaked_bytes": 3278099, + "total_allocations": 944654 + }, + "builder_sign_jpeg_add_ingredient_from_archive": { + "peak_bytes": 14075511, + "leaked_bytes": 3421125, + "total_allocations": 1753642 + }, + "builder_ingredient_archive_roundtrip": { + "peak_bytes": 14222976, + "leaked_bytes": 3419885, + "total_allocations": 2609017 + }, + "builder_sign_jpeg_two_ingredient_archives": { + "peak_bytes": 14075190, + "leaked_bytes": 3421103, + "total_allocations": 2161232 }, "reader_error_no_manifest": { - "peak_bytes": 3474039, - "leaked_bytes": 3232411, - "total_allocations": 303795 + "peak_bytes": 3504260, + "leaked_bytes": 3263283, + "total_allocations": 178873 }, "builder_error_invalid_manifest": { - "peak_bytes": 3271093, - "leaked_bytes": 3211670, - "total_allocations": 120613 + "peak_bytes": 3292723, + "leaked_bytes": 3235293, + "total_allocations": 96622 }, "reader_string_apis": { - "peak_bytes": 3888863, - "leaked_bytes": 3254426, - "total_allocations": 2806719 + "peak_bytes": 3915891, + "leaked_bytes": 3284213, + "total_allocations": 1185492 }, "fork_reader_collect": { - "peak_bytes": 3760272, - "leaked_bytes": 3261839, - "total_allocations": 1615850 + "peak_bytes": 3789318, + "leaked_bytes": 3291267, + "total_allocations": 705123 }, "fork_contended_mutex": { - "peak_bytes": 7585897, - "leaked_bytes": 3392170, - "total_allocations": 82616370 + "peak_bytes": 7617864, + "leaked_bytes": 3421711, + "total_allocations": 33591148 }, "fork_thread_local_orphan": { - "peak_bytes": 3845601, - "leaked_bytes": 3348583, - "total_allocations": 1681204 + "peak_bytes": 3876269, + "leaked_bytes": 3379616, + "total_allocations": 731511 }, "fork_gc_cycle": { - "peak_bytes": 3760522, - "leaked_bytes": 3261588, - "total_allocations": 1620079 + "peak_bytes": 3790340, + "leaked_bytes": 3291400, + "total_allocations": 706802 }, "fork_parent_frees_after_fork": { - "peak_bytes": 5959081, - "leaked_bytes": 3260442, - "total_allocations": 30560082 + "peak_bytes": 5989167, + "leaked_bytes": 3289951, + "total_allocations": 12461253 }, "fork_child_sys_exit": { - "peak_bytes": 3760808, - "leaked_bytes": 3261849, - "total_allocations": 1624602 + "peak_bytes": 3789334, + "leaked_bytes": 3291456, + "total_allocations": 708625 }, "fork_stream_cleanup": { - "peak_bytes": 3382176, - "leaked_bytes": 3210242, - "total_allocations": 111383 - }, - "reader_manifest_data_context": { - "peak_bytes": 7575548, - "leaked_bytes": 3399851, - "total_allocations": 1414270 + "peak_bytes": 3402893, + "leaked_bytes": 3230663, + "total_allocations": 93141 } } \ No newline at end of file diff --git a/tests/perf/run_profile.py b/tests/perf/run_profile.py index afeb4f26..16640fac 100644 --- a/tests/perf/run_profile.py +++ b/tests/perf/run_profile.py @@ -197,8 +197,8 @@ def _write_github_summary(results: dict, baseline: dict) -> None: f"Iterations: {ITERATIONS} · threshold: +{(THRESHOLD - 1) * 100:.0f}%" f"{f' · env: {PERF_ENV}' if PERF_ENV else ''}", "", - "| scenario | peak | leaked | allocs | peak Δ% | leaked Δ% | status |", - "|----------|------|--------|--------|---------|-----------|--------|", + "| scenario | peak | allocs | peak Δ% | memory used Δ% | status |", + "|----------|------|--------|---------|----------------|--------|", ] for name, m in results.items(): b = baseline.get(name, {}) if baseline else {} @@ -210,7 +210,7 @@ def _write_github_summary(results: dict, baseline: dict) -> None: ) status = "REGRESSED" if regressed else "ok" lines.append( - f"| {name} | {_fmt(m['peak_bytes'])} | {_fmt(m['leaked_bytes'])} " + f"| {name} | {_fmt(m['peak_bytes'])} " f"| {m['total_allocations']} | {_delta_pct(m['peak_bytes'], peak_base)} " f"| {_delta_pct(m['leaked_bytes'], leaked_base)} | {status} |" ) From 2129513e5f1dd4988a49761582bd90605ed89731 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 30 Jun 2026 20:20:04 -0700 Subject: [PATCH 5/9] fix: Update baseline --- Makefile | 16 ++++++++++++- .../python-3.10-slim-perf-Dockerfile | 15 ++++++++---- .../python-3.12-slim-perf-Dockerfile | 15 ++++++++---- tests/perf/scenarios.py | 24 ++++++++++++++++++- 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 8a192083..e1762040 100644 --- a/Makefile +++ b/Makefile @@ -150,9 +150,23 @@ SCENARIO ?= SCENARIO_ARG := $(if $(SCENARIO),--scenario $(SCENARIO),) # In CI, use en vars to write the report to the job run GH_SUMMARY_MOUNT := $(if $(GITHUB_STEP_SUMMARY),-v $(GITHUB_STEP_SUMMARY):$(GITHUB_STEP_SUMMARY),) +# Build the perf Docker image only if it is missing. The repo is bind-mounted at +# run time and the Dockerfile only COPYs requirements*.txt, so latest Python code +# is picked up without a rebuild; rebuild is only needed when deps/Dockerfile +# change (use perf-image-rebuild for that). +.PHONY: perf-image +perf-image: + @docker image inspect c2pa-memray-$(PERF_ENV) >/dev/null 2>&1 || \ + docker build -f tests/perf/Dockerfiles/$(PERF_ENV)-perf-Dockerfile -t c2pa-memray-$(PERF_ENV) . + +# Force a clean rebuild of the memray perf Docker image +.PHONY: perf-image-rebuild +perf-image-rebuild: + docker build --no-cache --pull -f tests/perf/Dockerfiles/$(PERF_ENV)-perf-Dockerfile -t c2pa-memray-$(PERF_ENV) . + +# Runs memory benchmarks. Pre-requisite: Docker image built using `make perf-image-rebuild`. .PHONY: memory-use-bench memory-use-bench: - docker build -f tests/perf/Dockerfiles/$(PERF_ENV)-perf-Dockerfile -t c2pa-memray-$(PERF_ENV) . docker run --rm -v $(PWD):/workspace $(GH_SUMMARY_MOUNT) -e PYTHONPATH=/workspace/src -e PERF_ENV=$(PERF_ENV) -e MEMRAY_ITERATIONS=$(MEMRAY_ITERATIONS) -e MEMRAY_THRESHOLD=$(MEMRAY_THRESHOLD) -e GITHUB_TOKEN -e GITHUB_STEP_SUMMARY c2pa-memray-$(PERF_ENV) python -m tests.perf.run_profile $(SCENARIO_ARG) $(PERF_ARGS) @echo "" @echo "Reports written to tests/perf/reports/" diff --git a/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile b/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile index b6f37f90..ee9488c9 100644 --- a/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile +++ b/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile @@ -2,13 +2,18 @@ FROM python:3.10.20-slim-bookworm WORKDIR /workspace -# libunwind for memray native stack unwinding. -# python3-dbg + libc6-dbg supply the debug symbols memray needs to resolve -# native (C) frames; without them --native flamegraphs warn "No debug -# information was found for the Python interpreter" and show unnamed frames. +# libunwind for memray native stack unwinding; libc6-dbg gives memray the glibc +# debug symbols to name libc native frames. +# NOTE: no python3-dbg here. The python:3.x-slim base ships its OWN interpreter +# at /usr/local (built by python.org, stripped), while Debian's python3-dbg is +# for the distro python (a different 3.11 build), so it cannot resolve this +# interpreter. That mismatch leaves the harmless "No debug information was found +# for the Python interpreter" warning on --native runs: interpreter C frames go +# unnamed, but all metrics (peak/leaked/allocs) are unaffected. Resolving it +# would mean changing the base image (see ubuntu-*-perf-Dockerfile, which use +# the distro python and so can use python3-dbg). RUN apt-get update && apt-get install -y --no-install-recommends \ libunwind-dev \ - python3-dbg \ libc6-dbg \ ca-certificates \ && rm -rf /var/lib/apt/lists/* diff --git a/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile b/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile index e98c0fd4..62bf988e 100644 --- a/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile +++ b/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile @@ -2,13 +2,18 @@ FROM python:3.12.13-slim-bookworm WORKDIR /workspace -# libunwind for memray native stack unwinding. -# python3-dbg + libc6-dbg supply the debug symbols memray needs to resolve -# native (C) frames; without them --native flamegraphs warn "No debug -# information was found for the Python interpreter" and show unnamed frames. +# libunwind for memray native stack unwinding; libc6-dbg gives memray the glibc +# debug symbols to name libc native frames. +# NOTE: no python3-dbg here. The python:3.x-slim base ships its OWN interpreter +# at /usr/local (built by python.org, stripped), while Debian's python3-dbg is +# for the distro python (a different 3.11 build), so it cannot resolve this +# interpreter. That mismatch leaves the harmless "No debug information was found +# for the Python interpreter" warning on --native runs: interpreter C frames go +# unnamed, but all metrics (peak/leaked/allocs) are unaffected. Resolving it +# would mean changing the base image (see ubuntu-*-perf-Dockerfile, which use +# the distro python and so can use python3-dbg). RUN apt-get update && apt-get install -y --no-install-recommends \ libunwind-dev \ - python3-dbg \ libc6-dbg \ ca-certificates \ && rm -rf /var/lib/apt/lists/* diff --git a/tests/perf/scenarios.py b/tests/perf/scenarios.py index 0e367fb0..d7c5f493 100644 --- a/tests/perf/scenarios.py +++ b/tests/perf/scenarios.py @@ -15,6 +15,7 @@ import os import sys import threading +import time from concurrent.futures import ThreadPoolExecutor from pathlib import Path from types import SimpleNamespace @@ -76,6 +77,11 @@ # Scenario name for progress output, set per-run by run_profile.py via the env. _SCENARIO = os.environ.get("PERF_SCENARIO", "") +# When set (by the --timing pass in run_profile.py), _iterate records the wall +# time of each iteration's loop body and writes the samples to this path on +# exhaustion. Unset for the memray pass and normal runs, so they pay nothing. +_TIMING_OUT = os.environ.get("PERF_TIMING_OUT", "") + def _iterate(n: int): """Yield range(n), printing a progress line to stderr ~every 10%. @@ -84,15 +90,31 @@ def _iterate(n: int): high iteration counts looks hung. The print is gated to ~10 lines total so it stays readable at N=100 and N=100000 alike, and writes to stderr so it never lands in the captured/parsed metrics output. + + When PERF_TIMING_OUT is set, each iteration's loop-body wall time is timed + and the per-iteration seconds are written (JSON list) to that path once the + generator is exhausted. A generator resumes its post-`yield` code when the + caller requests the next item, so stamping before `yield` and diffing after + the resume measures the caller's loop body, not this function's own work. """ step = max(1, n // 10) label = f"{_SCENARIO}: " if _SCENARIO else "" + timing = bool(_TIMING_OUT) + samples: list[float] = [] for i in range(n): if i % step == 0: print(f" {label}iter {i}/{n} ({i * 100 // n if n else 100}%)", file=sys.stderr, flush=True) - yield i + if timing: + start = time.perf_counter() + yield i + samples.append(time.perf_counter() - start) + else: + yield i print(f" {label}iter {n}/{n} (100%)", file=sys.stderr, flush=True) + if timing: + with open(_TIMING_OUT, "w", encoding="utf-8") as fh: + json.dump(samples, fh) def _make_signer() -> Signer: From a1be05500be4b1ecf61ce606613d1603506ef02f Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 30 Jun 2026 20:26:05 -0700 Subject: [PATCH 6/9] fix: One cleanup --- .../Dockerfiles/python-3.10-slim-perf-Dockerfile | 12 +----------- .../Dockerfiles/python-3.12-slim-perf-Dockerfile | 12 +----------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile b/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile index ee9488c9..100a082f 100644 --- a/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile +++ b/tests/perf/Dockerfiles/python-3.10-slim-perf-Dockerfile @@ -2,19 +2,9 @@ FROM python:3.10.20-slim-bookworm WORKDIR /workspace -# libunwind for memray native stack unwinding; libc6-dbg gives memray the glibc -# debug symbols to name libc native frames. -# NOTE: no python3-dbg here. The python:3.x-slim base ships its OWN interpreter -# at /usr/local (built by python.org, stripped), while Debian's python3-dbg is -# for the distro python (a different 3.11 build), so it cannot resolve this -# interpreter. That mismatch leaves the harmless "No debug information was found -# for the Python interpreter" warning on --native runs: interpreter C frames go -# unnamed, but all metrics (peak/leaked/allocs) are unaffected. Resolving it -# would mean changing the base image (see ubuntu-*-perf-Dockerfile, which use -# the distro python and so can use python3-dbg). +# libunwind-dev for memray native stack unwinding. RUN apt-get update && apt-get install -y --no-install-recommends \ libunwind-dev \ - libc6-dbg \ ca-certificates \ && rm -rf /var/lib/apt/lists/* diff --git a/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile b/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile index 62bf988e..03968dbc 100644 --- a/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile +++ b/tests/perf/Dockerfiles/python-3.12-slim-perf-Dockerfile @@ -2,19 +2,9 @@ FROM python:3.12.13-slim-bookworm WORKDIR /workspace -# libunwind for memray native stack unwinding; libc6-dbg gives memray the glibc -# debug symbols to name libc native frames. -# NOTE: no python3-dbg here. The python:3.x-slim base ships its OWN interpreter -# at /usr/local (built by python.org, stripped), while Debian's python3-dbg is -# for the distro python (a different 3.11 build), so it cannot resolve this -# interpreter. That mismatch leaves the harmless "No debug information was found -# for the Python interpreter" warning on --native runs: interpreter C frames go -# unnamed, but all metrics (peak/leaked/allocs) are unaffected. Resolving it -# would mean changing the base image (see ubuntu-*-perf-Dockerfile, which use -# the distro python and so can use python3-dbg). +# libunwind-dev for memray native stack unwinding. RUN apt-get update && apt-get install -y --no-install-recommends \ libunwind-dev \ - libc6-dbg \ ca-certificates \ && rm -rf /var/lib/apt/lists/* From 5fb875790f27f2ae84a9c610935062ee43181bbc Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 30 Jun 2026 20:27:03 -0700 Subject: [PATCH 7/9] fix: Second cleanup --- tests/perf/scenarios.py | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/tests/perf/scenarios.py b/tests/perf/scenarios.py index d7c5f493..0e367fb0 100644 --- a/tests/perf/scenarios.py +++ b/tests/perf/scenarios.py @@ -15,7 +15,6 @@ import os import sys import threading -import time from concurrent.futures import ThreadPoolExecutor from pathlib import Path from types import SimpleNamespace @@ -77,11 +76,6 @@ # Scenario name for progress output, set per-run by run_profile.py via the env. _SCENARIO = os.environ.get("PERF_SCENARIO", "") -# When set (by the --timing pass in run_profile.py), _iterate records the wall -# time of each iteration's loop body and writes the samples to this path on -# exhaustion. Unset for the memray pass and normal runs, so they pay nothing. -_TIMING_OUT = os.environ.get("PERF_TIMING_OUT", "") - def _iterate(n: int): """Yield range(n), printing a progress line to stderr ~every 10%. @@ -90,31 +84,15 @@ def _iterate(n: int): high iteration counts looks hung. The print is gated to ~10 lines total so it stays readable at N=100 and N=100000 alike, and writes to stderr so it never lands in the captured/parsed metrics output. - - When PERF_TIMING_OUT is set, each iteration's loop-body wall time is timed - and the per-iteration seconds are written (JSON list) to that path once the - generator is exhausted. A generator resumes its post-`yield` code when the - caller requests the next item, so stamping before `yield` and diffing after - the resume measures the caller's loop body, not this function's own work. """ step = max(1, n // 10) label = f"{_SCENARIO}: " if _SCENARIO else "" - timing = bool(_TIMING_OUT) - samples: list[float] = [] for i in range(n): if i % step == 0: print(f" {label}iter {i}/{n} ({i * 100 // n if n else 100}%)", file=sys.stderr, flush=True) - if timing: - start = time.perf_counter() - yield i - samples.append(time.perf_counter() - start) - else: - yield i + yield i print(f" {label}iter {n}/{n} (100%)", file=sys.stderr, flush=True) - if timing: - with open(_TIMING_OUT, "w", encoding="utf-8") as fh: - json.dump(samples, fh) def _make_signer() -> Signer: From d15b2da91846ec489cee75d0e5effa51b1221866 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Tue, 30 Jun 2026 20:28:58 -0700 Subject: [PATCH 8/9] fix: Runner --- .github/workflows/memory-benchmark.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/memory-benchmark.yml b/.github/workflows/memory-benchmark.yml index 5232e1ee..992587b6 100644 --- a/.github/workflows/memory-benchmark.yml +++ b/.github/workflows/memory-benchmark.yml @@ -26,6 +26,10 @@ jobs: steps: - uses: actions/checkout@v4 + # Build the perf image. + - name: Build memray perf image + run: make perf-image-rebuild + # Uses the Dockerfile environment for repeatable runs. - name: Run memray memory benchmark run: make memory-use-bench From 70bc6f7a2d184257a2e13a47b75f4e9b564b95e5 Mon Sep 17 00:00:00 2001 From: tmathern <60901087+tmathern@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:47:00 -0700 Subject: [PATCH 9/9] fix: cleanup obsolete benches --- Makefile | 4 -- requirements-dev.txt | 1 - tests/benchmark.py | 147 ------------------------------------------- 3 files changed, 152 deletions(-) delete mode 100644 tests/benchmark.py diff --git a/Makefile b/Makefile index e1762040..d4e7fffd 100644 --- a/Makefile +++ b/Makefile @@ -62,10 +62,6 @@ test: $(PYTHON) ./tests/test_unit_tests.py $(PYTHON) ./tests/test_unit_tests_threaded.py -# Runs benchmarks in the venv -benchmark: - $(PYTHON) -m pytest tests/benchmark.py -v - # Tests building and installing a local wheel package # Downloads required artifacts, builds the wheel, installs it, and verifies the installation test-local-wheel-build: diff --git a/requirements-dev.txt b/requirements-dev.txt index ae6c7a61..083439e7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,6 @@ toml==0.10.2 # For reading pyproject.toml files # Testing dependencies pytest>=8.1.0 -pytest-benchmark>=5.1.0 # for downloading the library artifacts requests>=2.0.0 diff --git a/tests/benchmark.py b/tests/benchmark.py deleted file mode 100644 index c576c272..00000000 --- a/tests/benchmark.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright 2025 Adobe. All rights reserved. -# This file is licensed to you under the Apache License, -# Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) -# or the MIT license (http://opensource.org/licenses/MIT), -# at your option. - -# Unless required by applicable law or agreed to in writing, -# this software is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR REPRESENTATIONS OF ANY KIND, either express or -# implied. See the LICENSE-MIT and LICENSE-APACHE files for the -# specific language governing permissions and limitations under -# each license. - -import os -import io -import json -import shutil -from c2pa import Reader, Builder, Signer, C2paSigningAlg, C2paSignerInfo - -PROJECT_PATH = os.getcwd() - -# Test paths -test_path = os.path.join(PROJECT_PATH, "tests", "fixtures", "C.jpg") -temp_dir = os.path.join(PROJECT_PATH, "tests", "temp") -output_path = os.path.join(temp_dir, "python_out.jpg") - -# Ensure temp directory exists -os.makedirs(temp_dir, exist_ok=True) - -manifestDefinition = { - "claim_generator": "python_test", - "claim_generator_info": [{ - "name": "python_test", - "version": "0.0.1", - }], - "format": "image/jpeg", - "title": "Python Test Image", - "ingredients": [], - "assertions": [ - { - "label": "c2pa.actions", - "data": { - "actions": [ - { - "action": "c2pa.created", - "digitalSourceType": "http://cv.iptc.org/newscodes/digitalsourcetype/digitalCreation" - } - ] - } - } - ] -} - -# Load private key and certificates -private_key = open("tests/fixtures/ps256.pem", "rb").read() -certs = open("tests/fixtures/ps256.pub", "rb").read() - -# Create a local Ps256 signer with certs and a timestamp server -signer_info = C2paSignerInfo( - alg=b"ps256", - sign_cert=certs, - private_key=private_key, - ta_url=b"http://timestamp.digicert.com" -) -signer = Signer.from_info(signer_info) -builder = Builder(manifestDefinition) - -# Load source image -source = open(test_path, "rb").read() - -# Run the benchmark: python -m pytest tests/benchmark.py -v - - -def test_files_read(): - """Benchmark reading a C2PA asset from a file.""" - with open(test_path, "rb") as f: - reader = Reader("image/jpeg", f) - result = reader.json() - reader.close() - assert result is not None - # Parse the JSON string into a dictionary - result_dict = json.loads(result) - # Additional assertions to verify the structure of the result - assert "active_manifest" in result_dict - assert "manifests" in result_dict - assert "validation_state" in result_dict - assert result_dict["validation_state"] == "Valid" - - -def test_streams_read(): - """Benchmark reading a C2PA asset from a stream.""" - with open(test_path, "rb") as file: - source = file.read() - reader = Reader("image/jpeg", io.BytesIO(source)) - result = reader.json() - reader.close() - assert result is not None - # Parse the JSON string into a dictionary - result_dict = json.loads(result) - # Additional assertions to verify the structure of the result - assert "active_manifest" in result_dict - assert "manifests" in result_dict - assert "validation_state" in result_dict - assert result_dict["validation_state"] == "Valid" - - -def test_files_build(): - """Benchmark building a C2PA asset from a file.""" - # Delete the output file if it exists - if os.path.exists(output_path): - os.remove(output_path) - with open(test_path, "rb") as source_file: - with open(output_path, "w+b") as dest_file: - builder.sign(signer, "image/jpeg", source_file, dest_file) - - -def test_streams_build(): - """Benchmark building a C2PA asset from a stream.""" - output = io.BytesIO(bytearray()) - with open(test_path, "rb") as source_file: - builder.sign(signer, "image/jpeg", source_file, output) - - -def test_files_reading(benchmark): - """Benchmark file-based reading.""" - benchmark(test_files_read) - - -def test_streams_reading(benchmark): - """Benchmark stream-based reading.""" - benchmark(test_streams_read) - - -def test_files_builder_signer_benchmark(benchmark): - """Benchmark file-based building.""" - benchmark(test_files_build) - - -def test_streams_builder_benchmark(benchmark): - """Benchmark stream-based building.""" - benchmark(test_streams_build) - - -def teardown_module(module): - """Clean up temporary files after all tests.""" - if os.path.exists(temp_dir): - shutil.rmtree(temp_dir)