From e428b06e9e969a930695350502f6bf018de23d32 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 16:31:44 +0000 Subject: [PATCH] Images: one lock on the image before its manifests and tags A manifest delete, a whole-image delete and image retention locked an image's manifests and then its tags (oci_dao::lock_image_for_retention, delete_manifest, lock_manifests). A push that moves a tag (oci_dao::set_tag) and a tag delete (oci_dao::delete_tag) locked the tag and then the manifest it points at or leaves. Two such transactions on one image could each wait for a row the other held; Postgres aborted one (40P01) and it answered 500, or failed its retention batch. The rows have no order all these paths can follow: a push learns which manifest a tag leaves only once it holds the tag. So every transaction that writes an image's manifests and tags now takes a transaction-scoped advisory lock on the image first (oci_dao::lock_image, as proxy_dao::lock_cached_package does for a cached package): the push and the import (commit_manifest), a proxy's cached manifest, the registry's and the admin API's tag and manifest deletes, each batch of a whole image's delete, retention's apply and set_imported_times. They take turns per image and never wait for each other's rows. The retry of a deadlocked admin delete and its second read of the image (#560) stay: an instance of an earlier release pushes without the lock during a rolling upgrade. Fixes #575 Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_012e6DJbtdvmZFvSvRWFDdP2 --- ...7df366472a839882e65c9a19032a45b45c127.json | 24 +++ crates/core/src/retention/images.rs | 5 + crates/db/src/oci_dao.rs | 65 ++++++- crates/db/tests/oci_images.rs | 158 +++++++++++++++++- crates/server/src/admin/images.rs | 28 +++- crates/server/src/images/push.rs | 4 + crates/server/tests/registry.rs | 114 ++++++++++++- docs/changes/2.2.0/575.fixed.md | 8 + 8 files changed, 388 insertions(+), 18 deletions(-) create mode 100644 .sqlx/query-36679e9d233ab4c1332c9ac1a447df366472a839882e65c9a19032a45b45c127.json create mode 100644 docs/changes/2.2.0/575.fixed.md diff --git a/.sqlx/query-36679e9d233ab4c1332c9ac1a447df366472a839882e65c9a19032a45b45c127.json b/.sqlx/query-36679e9d233ab4c1332c9ac1a447df366472a839882e65c9a19032a45b45c127.json new file mode 100644 index 00000000..428c34a6 --- /dev/null +++ b/.sqlx/query-36679e9d233ab4c1332c9ac1a447df366472a839882e65c9a19032a45b45c127.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtextextended('forge.oci_image.' || $2, $1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "36679e9d233ab4c1332c9ac1a447df366472a839882e65c9a19032a45b45c127" +} diff --git a/crates/core/src/retention/images.rs b/crates/core/src/retention/images.rs index db30529c..6e7e9944 100644 --- a/crates/core/src/retention/images.rs +++ b/crates/core/src/retention/images.rs @@ -1233,6 +1233,11 @@ pub async fn apply_images( } decided = Some(decided_now(&mut tx, repository_id, image, fence, batch).await?); } + if !dry_run { + // Whether or not the image was re-evaluated under its lock: the + // deletes below take a tag before its manifest. + oci_dao::lock_image(&mut tx, repository_id, image).await?; + } for &d in batch { let still = decided.as_ref().is_none_or(|now| now.contains(d)); let applied = diff --git a/crates/db/src/oci_dao.rs b/crates/db/src/oci_dao.rs index 6079c170..ad939f23 100644 --- a/crates/db/src/oci_dao.rs +++ b/crates/db/src/oci_dao.rs @@ -9,7 +9,7 @@ use chrono::{DateTime, Utc}; use serde_json::Value as JsonValue; -use sqlx::{PgExecutor, PgPool, Postgres, Transaction}; +use sqlx::{PgConnection, PgExecutor, PgPool, Postgres, Transaction}; use uuid::Uuid; use crate::blob_dao; @@ -546,6 +546,38 @@ pub async fn sweep_link( // ── Manifests and tags ────────────────────────────────────────────────────── +/// Lock an image for the rest of the caller's transaction: a +/// transaction-scoped advisory lock on the repository and the image's name +/// (as [`crate::proxy_dao::lock_cached_package`] locks a cached package). +/// +/// Every transaction that writes an image's manifests and tags takes this +/// lock **before its first row lock on either**, so two of them never hold +/// row locks at once and cannot wait for each other (#575). The rows alone +/// have no order all of them can follow: a push locks a tag before it knows +/// the manifest the tag leaves ([`set_tag`]), a tag delete likewise +/// ([`delete_tag`]), while a manifest delete and image retention start from +/// the manifests and go on to their tags ([`lock_image_for_retention`], +/// [`delete_manifest`], [`lock_manifests`]). +/// +/// Taking it a second time in a transaction is free. Taking it after a row +/// lock on a manifest or a tag of the image brings the deadlock back; a +/// transaction never locks two images. Two names that hash alike share a +/// lock, which only makes their writers take turns. +pub async fn lock_image( + conn: &mut PgConnection, + repository_id: i64, + image: &str, +) -> sqlx::Result<()> { + sqlx::query!( + "SELECT pg_advisory_xact_lock(hashtextextended('forge.oci_image.' || $2, $1))", + repository_id, + image, + ) + .execute(&mut *conn) + .await?; + Ok(()) +} + /// A stored manifest. #[derive(Debug, Clone)] pub struct ManifestRow { @@ -700,6 +732,9 @@ pub async fn set_imported_times( downloaded_at: Option>, ) -> sqlx::Result<()> { let mut tx = crate::begin(pool).await?; + // A manifest and then its tag: under the image's lock, like every write + // of the two. + lock_image(&mut tx, repository_id, image).await?; sqlx::query!( r#" UPDATE oci_manifest @@ -939,6 +974,10 @@ pub enum TagWrite { /// only when `may_move` is true. The tag row is locked before it is read, so /// the immutability decision and the `untagged_since` bookkeeping of the /// manifest it leaves are exact under concurrent pushes. +/// +/// The tag is locked before the manifests it points at and leaves, the +/// reverse of a manifest delete: the caller's transaction must hold the +/// image's lock ([`lock_image`]), taken before its first row lock. pub async fn set_tag( tx: &mut Transaction<'_, Postgres>, repository_id: i64, @@ -1048,12 +1087,18 @@ async fn refresh_untagged( } /// Delete a tag (its manifest stays). `false` when there was no such tag. +/// +/// The tag goes before its manifest is updated, the reverse of a manifest +/// delete, so the image is locked first ([`lock_image`]): this must be the +/// transaction's first lock on the image's manifests and tags, or follow +/// its [`lock_image`]. pub async fn delete_tag( tx: &mut Transaction<'_, Postgres>, repository_id: i64, image: &str, tag: &str, ) -> sqlx::Result { + lock_image(tx, repository_id, image).await?; let manifest_id = sqlx::query_scalar!( r#" DELETE FROM oci_tag @@ -1078,6 +1123,7 @@ pub async fn delete_tag( /// Delete a manifest and every tag pointing at it, releasing the manifest's /// own reference. Its config and layers stay linked to the image until the /// sweeper finds them unreferenced. `false` when there was no such manifest. +/// The image is locked first ([`lock_image`]), as for [`delete_tag`]. pub async fn delete_manifest( tx: &mut Transaction<'_, Postgres>, repository_id: i64, @@ -1085,6 +1131,7 @@ pub async fn delete_manifest( digest: &str, grace: chrono::Duration, ) -> sqlx::Result { + lock_image(tx, repository_id, image).await?; let Some(id) = sqlx::query_scalar!( r#" SELECT id FROM oci_manifest @@ -1151,6 +1198,8 @@ pub async fn image_manifest_ids( /// Lock (`FOR UPDATE`) those of the manifests `ids` that still exist, in /// id order as [`lock_image_for_retention`] locks them, and return them. +/// Their tags are deleted next: the caller's transaction must hold their +/// image's lock ([`lock_image`]). pub async fn lock_manifests( tx: &mut Transaction<'_, Postgres>, ids: &[i64], @@ -2036,13 +2085,16 @@ pub async fn image_for_delete_preview( /// Lock (`FOR UPDATE`) and load one image's manifests and tags, as /// [`retention_manifests`] and [`retention_tags`] load a repository's: for /// image retention to re-evaluate its decisions against the image's current -/// state, which stays put until the caller's transaction ends. Manifests are -/// locked before tags, in id and tag order. +/// state, which stays put until the caller's transaction ends. The image is +/// locked first ([`lock_image`]), then its manifests before its tags, in id +/// and tag order: a push or a tag delete, which lock a tag before its +/// manifest, wait for the image's lock instead of deadlocking with this. pub async fn lock_image_for_retention( tx: &mut Transaction<'_, Postgres>, repository_id: i64, image: &str, ) -> sqlx::Result<(Vec, Vec)> { + lock_image(tx, repository_id, image).await?; let manifests = sqlx::query_as!( RetentionManifest, r#" @@ -2080,7 +2132,8 @@ pub async fn lock_image_for_retention( } /// Delete `tag` only if it still points at `manifest_id` (retention decided -/// on that). `false`: it moved or is gone. +/// on that). `false`: it moved or is gone. The image is locked first +/// ([`lock_image`]), as for [`delete_tag`]. pub async fn delete_tag_if( tx: &mut Transaction<'_, Postgres>, repository_id: i64, @@ -2088,6 +2141,7 @@ pub async fn delete_tag_if( tag: &str, manifest_id: i64, ) -> sqlx::Result { + lock_image(tx, repository_id, image).await?; let current = sqlx::query_scalar!( r#" SELECT manifest_id FROM oci_tag @@ -2108,7 +2162,8 @@ pub async fn delete_tag_if( /// Delete an untagged manifest that no other manifest of its image lists as /// a child, re-checked under the row lock (a tag or an index pushed since -/// retention decided keeps it). `false`: kept. +/// retention decided keeps it). `false`: kept. The caller's transaction +/// must hold the image's lock ([`lock_image`]). pub async fn delete_untagged_manifest( tx: &mut Transaction<'_, Postgres>, repository_id: i64, diff --git a/crates/db/tests/oci_images.rs b/crates/db/tests/oci_images.rs index 62c96058..5f19230c 100644 --- a/crates/db/tests/oci_images.rs +++ b/crates/db/tests/oci_images.rs @@ -1,5 +1,6 @@ //! `oci_dao`'s image listings against a real Postgres: reading every image -//! of a repository a batch at a time by name (#596). +//! of a repository a batch at a time by name (#596); and the lock every +//! write of an image's manifests and tags takes first (#575). mod common; @@ -96,3 +97,158 @@ async fn reading_every_image_by_name_reads_each_manifest_a_few_times() { "listing {IMAGES} images read {read} manifest rows" ); } + +/// Wait until the session `pid` is blocked on a lock. Panics after 10 s. +async fn wait_for_lock_wait(pool: &sqlx::PgPool, pid: i32) { + for _ in 0..500 { + let waiting: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity \ + WHERE pid = $1 AND wait_event_type = 'Lock')", + ) + .bind(pid) + .fetch_one(pool) + .await + .unwrap(); + if waiting { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + panic!("session {pid} never waited for a lock"); +} + +async fn backend_pid(conn: &mut sqlx::PgConnection) -> i32 { + sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(conn) + .await + .unwrap() +} + +/// Image retention (and an administrator's manifest delete) locks an image's +/// manifests and then its tags; a tag delete and a push that moves a tag +/// lock the tag and then its manifest. In these opposite orders the two +/// transactions deadlocked and Postgres aborted the tag's writer (`40P01`, +/// #575). Both take the image's lock first: the writer waits for it. +#[tokio::test] +async fn a_tag_write_waits_for_an_image_locked_for_retention() { + let pool = pool().await; + let repo = repository_dao::insert_repository( + &pool, + NewRepository::hosted(format!("images-{}", unique())), + ) + .await + .unwrap() + .id; + let sha = format!("{:064x}", (u128::from(unique()) << 64) | 0x575); + sqlx::query( + "INSERT INTO blob (sha256, size, s3_key, state) VALUES ($1, 1, $1, 'committed') \ + ON CONFLICT DO NOTHING", + ) + .bind(&sha) + .execute(&pool) + .await + .unwrap(); + for (image, moves) in [("deleted", false), ("moved", true)] { + // `latest` on one manifest and `aaa`, the first tag in tag order, + // on another. + let mut manifests = Vec::new(); + for (n, tag) in ["latest", "aaa"].into_iter().enumerate() { + let id: i64 = sqlx::query_scalar( + r#" + INSERT INTO oci_manifest (repository_id, image, digest, blob_sha256, media_type, size) + VALUES ($1, $2, 'sha256:' || $3 || $4::text, $3, + 'application/vnd.oci.image.manifest.v1+json', 1) + RETURNING id + "#, + ) + .bind(repo) + .bind(image) + .bind(&sha) + .bind(n as i64) + .fetch_one(&pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO oci_tag (repository_id, image, tag, manifest_id) \ + VALUES ($1, $2, $3, $4)", + ) + .bind(repo) + .bind(image) + .bind(tag) + .bind(id) + .execute(&pool) + .await + .unwrap(); + manifests.push(id); + } + let (tagged, other) = (manifests[0], manifests[1]); + + // Retention holds the image's manifests and stops at its first tag, + // which another transaction holds. + let mut blocker = pool.begin().await.unwrap(); + sqlx::query( + "SELECT 1 FROM oci_tag WHERE repository_id = $1 AND image = $2 AND tag = 'aaa' \ + FOR UPDATE", + ) + .bind(repo) + .bind(image) + .execute(&mut *blocker) + .await + .unwrap(); + let mut retention = pool.begin().await.unwrap(); + let retention_pid = backend_pid(&mut retention).await; + let retention = tokio::spawn(async move { + let locked = oci_dao::lock_image_for_retention(&mut retention, repo, image).await; + (retention, locked) + }); + wait_for_lock_wait(&pool, retention_pid).await; + + // The writer of `latest` starts meanwhile: it locked the tag, and + // then waited for the manifest retention holds. + let mut writer = pool.begin().await.unwrap(); + let writer_pid = backend_pid(&mut writer).await; + let writer = tokio::spawn(async move { + let written = if moves { + // As a push does (`commit_manifest`). + match oci_dao::lock_image(&mut writer, repo, image).await { + Ok(()) => oci_dao::set_tag(&mut writer, repo, image, "latest", other, true) + .await + .map(|write| write == oci_dao::TagWrite::Moved { previous: tagged }), + Err(e) => Err(e), + } + } else { + oci_dao::delete_tag(&mut writer, repo, image, "latest").await + }; + (writer, written) + }); + wait_for_lock_wait(&pool, writer_pid).await; + + // Retention goes on to the tags, `latest` among them. + blocker.rollback().await.unwrap(); + let (retention, locked) = retention.await.unwrap(); + let (tags, locked) = locked.expect("retention locks the image"); + assert_eq!((tags.len(), locked.len()), (2, 2), "{image}"); + retention.commit().await.unwrap(); + let (writer, written) = writer.await.unwrap(); + assert!(written.expect("the tag's writer is not aborted"), "{image}"); + writer.commit().await.unwrap(); + + let latest: Option = sqlx::query_scalar( + "SELECT manifest_id FROM oci_tag \ + WHERE repository_id = $1 AND image = $2 AND tag = 'latest'", + ) + .bind(repo) + .bind(image) + .fetch_optional(&pool) + .await + .unwrap(); + assert_eq!(latest, moves.then_some(other), "{image}"); + let untagged: bool = + sqlx::query_scalar("SELECT untagged_since IS NOT NULL FROM oci_manifest WHERE id = $1") + .bind(tagged) + .fetch_one(&pool) + .await + .unwrap(); + assert!(untagged, "{image}: the manifest `latest` left"); + } +} diff --git a/crates/server/src/admin/images.rs b/crates/server/src/admin/images.rs index ed364c44..715cdb38 100644 --- a/crates/server/src/admin/images.rs +++ b/crates/server/src/admin/images.rs @@ -324,14 +324,17 @@ async fn delete_tag( /// How many manifests deleting a whole image deletes per transaction. const IMAGE_DELETE_BATCH: usize = 100; -/// Delete the manifest `digest` with what only it kept. The image's -/// manifests and tags are locked first, so a push cannot tag or list a +/// Delete the manifest `digest` with what only it kept. The image and then +/// its manifests and tags are locked first, so a push cannot tag or list a /// manifest this decided to delete before it is gone. /// -/// The locking statement alone does not show what to decide on: it waits for -/// a push that holds one of the manifests (an index listing it holds it -/// `FOR SHARE`), and its snapshot predates what that push committed -/// meanwhile, the index included (#560). So the image is read a second time, +/// A push holds the image's lock, and this waits for it before it reads +/// anything (#575). An instance of an earlier release, during a rolling +/// upgrade, pushes without that lock, and then the locking statement alone +/// does not show what to decide on: it waits for a push that holds one of +/// the manifests (an index listing it holds it `FOR SHARE`), and its +/// snapshot predates what that push committed meanwhile, the index included +/// (#560). So the image is read a second time, /// under the locks: whatever lists or tags a manifest the first statement /// locked had to commit before the lock was granted, and the second read /// sees it. A manifest only the second read returns arrived in between; what @@ -378,7 +381,9 @@ async fn delete_image( return Ok(deleted); } for batch in ids.chunks(IMAGE_DELETE_BATCH) { - let done = retrying_a_deadlock(image, || delete_batch(pool, batch, grace)).await?; + let done = + retrying_a_deadlock(image, || delete_batch(pool, repo_id, image, batch, grace)) + .await?; deleted.manifests += done.manifests; deleted.tags += done.tags; } @@ -388,10 +393,13 @@ async fn delete_image( /// Lock and delete one batch of [`delete_image`], in a transaction. async fn delete_batch( pool: &sqlx::PgPool, + repo_id: i64, + image: &str, batch: &[i64], grace: chrono::Duration, ) -> sqlx::Result { let mut tx = forge_db::begin(pool).await?; + oci_dao::lock_image(&mut tx, repo_id, image).await?; let locked = oci_dao::lock_manifests(&mut tx, batch).await?; let done = oci_dao::delete_locked_manifests(&mut tx, &locked, grace).await?; tx.commit().await?; @@ -401,8 +409,10 @@ async fn delete_batch( /// Run `transaction`, and once more when it deadlocked (`40P01`). A delete /// locks the manifests in id order and a push the manifests its index lists /// in digest order, so the two can wait for each other; Postgres then aborts -/// one side. The delete's transaction rolled back whole, so running it again -/// is safe, as for a repository delete. +/// one side. The image's lock, which both take first, rules that out +/// (#575), except against a push by an instance of an earlier release +/// during a rolling upgrade. The delete's transaction rolled back whole, so +/// running it again is safe, as for a repository delete. async fn retrying_a_deadlock(image: &str, transaction: F) -> sqlx::Result where F: Fn() -> Fut, diff --git a/crates/server/src/images/push.rs b/crates/server/src/images/push.rs index c35233ab..d2044c85 100644 --- a/crates/server/src/images/push.rs +++ b/crates/server/src/images/push.rs @@ -532,6 +532,9 @@ impl Registry { .role_filters_for_mutation(principal, &t.repo, PackageMutation::Publish) .await; let mut tx = forge_db::begin(&self.inner.pool).await?; + // Before the first lock on a manifest or a tag: a manifest delete + // takes them in the reverse order of the tags' writes below (#575). + oci_dao::lock_image(&mut tx, t.repo.id, &t.image).await?; let linked: HashSet = oci_dao::lock_links(&mut tx, t.repo.id, &t.image, &required_blobs) .await? @@ -878,6 +881,7 @@ impl Registry { let stored = self.inner.cas.commit(staged).await?; let mut tx = forge_db::begin(&self.inner.pool).await?; super::proxy::still_upstream(&mut tx, &t.repo).await?; + oci_dao::lock_image(&mut tx, t.repo.id, &t.image).await?; let (manifest_id, _) = oci_dao::insert_manifest( &mut tx, NewManifest { diff --git a/crates/server/tests/registry.rs b/crates/server/tests/registry.rs index 0a4edce6..93c89a45 100644 --- a/crates/server/tests/registry.rs +++ b/crates/server/tests/registry.rs @@ -1542,6 +1542,10 @@ async fn admins_delete_an_image_of_many_manifests_in_batches() { assert_eq!(oci.head(&other, "manifests/v0").await.status(), 200); } +/// The statement that takes an image's lock (`oci_dao::lock_image`), as +/// `common::wait_for_lock_wait` matches it. +const IMAGE_LOCK: &str = "%pg_advisory_xact_lock%forge.oci_image.%"; + /// An index listing the manifests `children` as platform manifests. fn index_of(children: &[&str], note: &str) -> Vec { json!({ @@ -1575,7 +1579,9 @@ async fn manifest_id(srv: &TestServer, repo_id: i64, image: &str, digest: &str) /// that manifest's lock; the delete's locking statement, whose snapshot /// predates the push, never returns the new index. The delete used to decide /// on that statement's rows and deleted the platform manifest `v2` lists; it -/// reads the image again under the locks and keeps it. +/// reads the image again under the locks and keeps it. Since #575 the delete +/// waits for the push earlier, at the image's lock, and reads the image only +/// once the push committed. #[tokio::test] async fn an_admin_delete_keeps_a_platform_manifest_an_index_pushed_meanwhile_lists() { let srv = common::start().await; @@ -1625,7 +1631,8 @@ async fn an_admin_delete_keeps_a_platform_manifest_an_index_pushed_meanwhile_lis }); common::wait_for_lock_wait(&srv.pool, &["%INSERT INTO oci_tag%ON CONFLICT%"]).await; - // The delete of v1's index waits for the platform manifest. + // The delete of v1's index waits for the push, which holds the image's + // lock. let (u, p) = srv.admin_basic(); let delete = tokio::spawn({ let request = srv @@ -1635,7 +1642,7 @@ async fn an_admin_delete_keeps_a_platform_manifest_an_index_pushed_meanwhile_lis .query(&[("image", "app".to_string()), ("reference", sha(&old_index))]); async move { request.send().await.unwrap() } }); - common::wait_for_lock_wait(&srv.pool, &["%FROM oci_manifest m%FOR UPDATE OF m%"]).await; + common::wait_for_lock_wait(&srv.pool, &[IMAGE_LOCK]).await; // The push commits its index and its tag; only then is the delete // granted the lock. @@ -1759,6 +1766,107 @@ async fn an_admin_delete_that_deadlocks_with_a_push_runs_again() { } } +/// An administrator deletes a manifest of an image while a tag of the image +/// is written: CI pushes `latest` again, or the tag is deleted, from the +/// admin API or by a registry client (#575). The delete locked the image's +/// manifests and then its tags; the tag's writer locked the tag and then the +/// manifest it points at, or the one it leaves. Each then waited for a row +/// the other held, Postgres aborted the writer, and it answered 500. Both now +/// take the image's lock first, so the writer waits for the delete to commit. +#[tokio::test] +async fn an_image_delete_and_a_tag_write_wait_for_each_other_instead_of_deadlocking() { + let srv = common::start().await; + let repo = repo_name("dock"); + let created = create_repo( + &srv, + json!({"name": repo, "kind": "hosted", "format": "docker"}), + ) + .await; + let repo_id = created["id"].as_i64().unwrap(); + let oci = admin_oci(&srv).await; + let (u, p) = srv.admin_basic(); + let images_url = format!("{}/admin/repositories/{repo}/images", srv.base_url); + for writer in ["push", "admin-tag-delete", "registry-tag-delete"] { + let name = format!("{repo}/{writer}"); + let layer = format!("lock order {writer} {}", unique()).into_bytes(); + // What `latest` points at, what the push moves it to, and the + // manifest the administrator deletes, the first in tag order. + oci.push_image(&name, "latest", "amd64", &layer).await; + let next = image(&config("arm64"), &[&layer]); + oci.push_image(&name, "next", "arm64", &layer).await; + let doomed = oci.push_image(&name, "aaa", "s390x", &layer).await; + + // The delete holds the image's lock and its manifests, and stops at + // its first tag, which another transaction holds. + let mut blocker = srv.pool.begin().await.unwrap(); + sqlx::query( + "SELECT 1 FROM oci_tag WHERE repository_id = $1 AND image = $2 AND tag = 'aaa' \ + FOR UPDATE", + ) + .bind(repo_id) + .bind(writer) + .execute(&mut *blocker) + .await + .unwrap(); + let delete = tokio::spawn({ + let request = srv + .client() + .delete(&images_url) + .basic_auth(&u, Some(&p)) + .query(&[("image", writer), ("reference", doomed.as_str())]); + async move { request.send().await.unwrap() } + }); + common::wait_for_lock_wait(&srv.pool, &["%FROM oci_tag t%FOR UPDATE OF t%"]).await; + + // The writer of `latest` starts meanwhile and waits for the delete. + let request = match writer { + "push" => srv + .client() + .put(format!("{}/v2/{name}/manifests/latest", srv.base_url)) + .header(AUTHORIZATION, oci.bearer(&name, "pull,push").await) + .header(CONTENT_TYPE, OCI_MANIFEST) + .body(next.clone()), + "admin-tag-delete" => srv + .client() + .delete(&images_url) + .basic_auth(&u, Some(&p)) + .query(&[("image", writer), ("reference", "latest")]), + _ => srv + .client() + .delete(format!("{}/v2/{name}/manifests/latest", srv.base_url)) + .header(AUTHORIZATION, oci.bearer(&name, "delete").await), + }; + let write = tokio::spawn(async move { request.send().await.unwrap().status() }); + common::wait_for_lock_wait(&srv.pool, &[IMAGE_LOCK]).await; + + // The delete goes on to the tags; neither is aborted. + blocker.rollback().await.unwrap(); + let resp = delete.await.unwrap(); + assert_eq!(resp.status(), 200, "{writer}"); + assert_eq!( + resp.json::().await.unwrap(), + json!({"manifests": 1, "tags": 1}), + "{writer}" + ); + let (written, latest) = match writer { + "push" => (201, 200), + "admin-tag-delete" => (200, 404), + _ => (202, 404), + }; + assert_eq!(write.await.unwrap(), written, "{writer}"); + let resp = oci.head(&name, "manifests/latest").await; + assert_eq!(resp.status(), latest, "{writer}"); + if writer == "push" { + assert_eq!( + resp.headers()["docker-content-digest"].to_str().unwrap(), + sha(&next) + ); + } + assert_eq!(oci.head(&name, "manifests/aaa").await.status(), 404); + assert_eq!(oci.head(&name, "manifests/next").await.status(), 200); + } +} + const HELM_CONFIG: &str = "application/vnd.cncf.helm.config.v1+json"; const HELM_CHART: &str = "application/vnd.cncf.helm.chart.content.v1.tar+gzip"; diff --git a/docs/changes/2.2.0/575.fixed.md b/docs/changes/2.2.0/575.fixed.md new file mode 100644 index 00000000..0c232382 --- /dev/null +++ b/docs/changes/2.2.0/575.fixed.md @@ -0,0 +1,8 @@ +Deleting a manifest or a whole image (from the web UI's image page, with +`DELETE /admin/repositories/{name}/images`, or by image retention) while a +push moved a tag of the same image, or while a tag of it was deleted, could +deadlock: the delete locked the image's manifests and then its tags, the +tag's writer the tag and then its manifest, and whichever Postgres aborted +answered 500 (a retention run failed its batch). Every write to an image's +manifests and tags now takes one lock on the image first, so they wait for +each other instead.