Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions crates/core/src/retention/images.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1233,6 +1233,11 @@ pub async fn apply_images(
}
decided = Some(decided_now(&mut tx, repository_id, image, fence, batch).await?);
}
if !dry_run {
// Whether or not the image was re-evaluated under its lock: the
// deletes below take a tag before its manifest.
oci_dao::lock_image(&mut tx, repository_id, image).await?;
}
for &d in batch {
let still = decided.as_ref().is_none_or(|now| now.contains(d));
let applied =
Expand Down
65 changes: 60 additions & 5 deletions crates/db/src/oci_dao.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

use chrono::{DateTime, Utc};
use serde_json::Value as JsonValue;
use sqlx::{PgExecutor, PgPool, Postgres, Transaction};
use sqlx::{PgConnection, PgExecutor, PgPool, Postgres, Transaction};
use uuid::Uuid;

use crate::blob_dao;
Expand Down Expand Up @@ -546,6 +546,38 @@ pub async fn sweep_link(

// ── Manifests and tags ──────────────────────────────────────────────────────

/// Lock an image for the rest of the caller's transaction: a
/// transaction-scoped advisory lock on the repository and the image's name
/// (as [`crate::proxy_dao::lock_cached_package`] locks a cached package).
///
/// Every transaction that writes an image's manifests and tags takes this
/// lock **before its first row lock on either**, so two of them never hold
/// row locks at once and cannot wait for each other (#575). The rows alone
/// have no order all of them can follow: a push locks a tag before it knows
/// the manifest the tag leaves ([`set_tag`]), a tag delete likewise
/// ([`delete_tag`]), while a manifest delete and image retention start from
/// the manifests and go on to their tags ([`lock_image_for_retention`],
/// [`delete_manifest`], [`lock_manifests`]).
///
/// Taking it a second time in a transaction is free. Taking it after a row
/// lock on a manifest or a tag of the image brings the deadlock back; a
/// transaction never locks two images. Two names that hash alike share a
/// lock, which only makes their writers take turns.
pub async fn lock_image(
conn: &mut PgConnection,
repository_id: i64,
image: &str,
) -> sqlx::Result<()> {
sqlx::query!(
"SELECT pg_advisory_xact_lock(hashtextextended('forge.oci_image.' || $2, $1))",
repository_id,
image,
)
.execute(&mut *conn)
.await?;
Ok(())
}

/// A stored manifest.
#[derive(Debug, Clone)]
pub struct ManifestRow {
Expand Down Expand Up @@ -700,6 +732,9 @@ pub async fn set_imported_times(
downloaded_at: Option<DateTime<Utc>>,
) -> sqlx::Result<()> {
let mut tx = crate::begin(pool).await?;
// A manifest and then its tag: under the image's lock, like every write
// of the two.
lock_image(&mut tx, repository_id, image).await?;
sqlx::query!(
r#"
UPDATE oci_manifest
Expand Down Expand Up @@ -939,6 +974,10 @@ pub enum TagWrite {
/// only when `may_move` is true. The tag row is locked before it is read, so
/// the immutability decision and the `untagged_since` bookkeeping of the
/// manifest it leaves are exact under concurrent pushes.
///
/// The tag is locked before the manifests it points at and leaves, the
/// reverse of a manifest delete: the caller's transaction must hold the
/// image's lock ([`lock_image`]), taken before its first row lock.
pub async fn set_tag(
tx: &mut Transaction<'_, Postgres>,
repository_id: i64,
Expand Down Expand Up @@ -1048,12 +1087,18 @@ async fn refresh_untagged(
}

/// Delete a tag (its manifest stays). `false` when there was no such tag.
///
/// The tag goes before its manifest is updated, the reverse of a manifest
/// delete, so the image is locked first ([`lock_image`]): this must be the
/// transaction's first lock on the image's manifests and tags, or follow
/// its [`lock_image`].
pub async fn delete_tag(
tx: &mut Transaction<'_, Postgres>,
repository_id: i64,
image: &str,
tag: &str,
) -> sqlx::Result<bool> {
lock_image(tx, repository_id, image).await?;
let manifest_id = sqlx::query_scalar!(
r#"
DELETE FROM oci_tag
Expand All @@ -1078,13 +1123,15 @@ pub async fn delete_tag(
/// Delete a manifest and every tag pointing at it, releasing the manifest's
/// own reference. Its config and layers stay linked to the image until the
/// sweeper finds them unreferenced. `false` when there was no such manifest.
/// The image is locked first ([`lock_image`]), as for [`delete_tag`].
pub async fn delete_manifest(
tx: &mut Transaction<'_, Postgres>,
repository_id: i64,
image: &str,
digest: &str,
grace: chrono::Duration,
) -> sqlx::Result<bool> {
lock_image(tx, repository_id, image).await?;
let Some(id) = sqlx::query_scalar!(
r#"
SELECT id FROM oci_manifest
Expand Down Expand Up @@ -1151,6 +1198,8 @@ pub async fn image_manifest_ids(

/// Lock (`FOR UPDATE`) those of the manifests `ids` that still exist, in
/// id order as [`lock_image_for_retention`] locks them, and return them.
/// Their tags are deleted next: the caller's transaction must hold their
/// image's lock ([`lock_image`]).
pub async fn lock_manifests(
tx: &mut Transaction<'_, Postgres>,
ids: &[i64],
Expand Down Expand Up @@ -2036,13 +2085,16 @@ pub async fn image_for_delete_preview(
/// Lock (`FOR UPDATE`) and load one image's manifests and tags, as
/// [`retention_manifests`] and [`retention_tags`] load a repository's: for
/// image retention to re-evaluate its decisions against the image's current
/// state, which stays put until the caller's transaction ends. Manifests are
/// locked before tags, in id and tag order.
/// state, which stays put until the caller's transaction ends. The image is
/// locked first ([`lock_image`]), then its manifests before its tags, in id
/// and tag order: a push or a tag delete, which lock a tag before its
/// manifest, wait for the image's lock instead of deadlocking with this.
pub async fn lock_image_for_retention(
tx: &mut Transaction<'_, Postgres>,
repository_id: i64,
image: &str,
) -> sqlx::Result<(Vec<RetentionTag>, Vec<RetentionManifest>)> {
lock_image(tx, repository_id, image).await?;
let manifests = sqlx::query_as!(
RetentionManifest,
r#"
Expand Down Expand Up @@ -2080,14 +2132,16 @@ pub async fn lock_image_for_retention(
}

/// Delete `tag` only if it still points at `manifest_id` (retention decided
/// on that). `false`: it moved or is gone.
/// on that). `false`: it moved or is gone. The image is locked first
/// ([`lock_image`]), as for [`delete_tag`].
pub async fn delete_tag_if(
tx: &mut Transaction<'_, Postgres>,
repository_id: i64,
image: &str,
tag: &str,
manifest_id: i64,
) -> sqlx::Result<bool> {
lock_image(tx, repository_id, image).await?;
let current = sqlx::query_scalar!(
r#"
SELECT manifest_id FROM oci_tag
Expand All @@ -2108,7 +2162,8 @@ pub async fn delete_tag_if(

/// Delete an untagged manifest that no other manifest of its image lists as
/// a child, re-checked under the row lock (a tag or an index pushed since
/// retention decided keeps it). `false`: kept.
/// retention decided keeps it). `false`: kept. The caller's transaction
/// must hold the image's lock ([`lock_image`]).
pub async fn delete_untagged_manifest(
tx: &mut Transaction<'_, Postgres>,
repository_id: i64,
Expand Down
158 changes: 157 additions & 1 deletion crates/db/tests/oci_images.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! `oci_dao`'s image listings against a real Postgres: reading every image
//! of a repository a batch at a time by name (#596).
//! of a repository a batch at a time by name (#596); and the lock every
//! write of an image's manifests and tags takes first (#575).

mod common;

Expand Down Expand Up @@ -96,3 +97,158 @@ async fn reading_every_image_by_name_reads_each_manifest_a_few_times() {
"listing {IMAGES} images read {read} manifest rows"
);
}

/// Wait until the session `pid` is blocked on a lock. Panics after 10 s.
async fn wait_for_lock_wait(pool: &sqlx::PgPool, pid: i32) {
for _ in 0..500 {
let waiting: bool = sqlx::query_scalar(
"SELECT EXISTS (SELECT 1 FROM pg_stat_activity \
WHERE pid = $1 AND wait_event_type = 'Lock')",
)
.bind(pid)
.fetch_one(pool)
.await
.unwrap();
if waiting {
return;
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
panic!("session {pid} never waited for a lock");
}

async fn backend_pid(conn: &mut sqlx::PgConnection) -> i32 {
sqlx::query_scalar("SELECT pg_backend_pid()")
.fetch_one(conn)
.await
.unwrap()
}

/// Image retention (and an administrator's manifest delete) locks an image's
/// manifests and then its tags; a tag delete and a push that moves a tag
/// lock the tag and then its manifest. In these opposite orders the two
/// transactions deadlocked and Postgres aborted the tag's writer (`40P01`,
/// #575). Both take the image's lock first: the writer waits for it.
#[tokio::test]
async fn a_tag_write_waits_for_an_image_locked_for_retention() {
let pool = pool().await;
let repo = repository_dao::insert_repository(
&pool,
NewRepository::hosted(format!("images-{}", unique())),
)
.await
.unwrap()
.id;
let sha = format!("{:064x}", (u128::from(unique()) << 64) | 0x575);
sqlx::query(
"INSERT INTO blob (sha256, size, s3_key, state) VALUES ($1, 1, $1, 'committed') \
ON CONFLICT DO NOTHING",
)
.bind(&sha)
.execute(&pool)
.await
.unwrap();
for (image, moves) in [("deleted", false), ("moved", true)] {
// `latest` on one manifest and `aaa`, the first tag in tag order,
// on another.
let mut manifests = Vec::new();
for (n, tag) in ["latest", "aaa"].into_iter().enumerate() {
let id: i64 = sqlx::query_scalar(
r#"
INSERT INTO oci_manifest (repository_id, image, digest, blob_sha256, media_type, size)
VALUES ($1, $2, 'sha256:' || $3 || $4::text, $3,
'application/vnd.oci.image.manifest.v1+json', 1)
RETURNING id
"#,
)
.bind(repo)
.bind(image)
.bind(&sha)
.bind(n as i64)
.fetch_one(&pool)
.await
.unwrap();
sqlx::query(
"INSERT INTO oci_tag (repository_id, image, tag, manifest_id) \
VALUES ($1, $2, $3, $4)",
)
.bind(repo)
.bind(image)
.bind(tag)
.bind(id)
.execute(&pool)
.await
.unwrap();
manifests.push(id);
}
let (tagged, other) = (manifests[0], manifests[1]);

// Retention holds the image's manifests and stops at its first tag,
// which another transaction holds.
let mut blocker = pool.begin().await.unwrap();
sqlx::query(
"SELECT 1 FROM oci_tag WHERE repository_id = $1 AND image = $2 AND tag = 'aaa' \
FOR UPDATE",
)
.bind(repo)
.bind(image)
.execute(&mut *blocker)
.await
.unwrap();
let mut retention = pool.begin().await.unwrap();
let retention_pid = backend_pid(&mut retention).await;
let retention = tokio::spawn(async move {
let locked = oci_dao::lock_image_for_retention(&mut retention, repo, image).await;
(retention, locked)
});
wait_for_lock_wait(&pool, retention_pid).await;

// The writer of `latest` starts meanwhile: it locked the tag, and
// then waited for the manifest retention holds.
let mut writer = pool.begin().await.unwrap();
let writer_pid = backend_pid(&mut writer).await;
let writer = tokio::spawn(async move {
let written = if moves {
// As a push does (`commit_manifest`).
match oci_dao::lock_image(&mut writer, repo, image).await {
Ok(()) => oci_dao::set_tag(&mut writer, repo, image, "latest", other, true)
.await
.map(|write| write == oci_dao::TagWrite::Moved { previous: tagged }),
Err(e) => Err(e),
}
} else {
oci_dao::delete_tag(&mut writer, repo, image, "latest").await
};
(writer, written)
});
wait_for_lock_wait(&pool, writer_pid).await;

// Retention goes on to the tags, `latest` among them.
blocker.rollback().await.unwrap();
let (retention, locked) = retention.await.unwrap();
let (tags, locked) = locked.expect("retention locks the image");
assert_eq!((tags.len(), locked.len()), (2, 2), "{image}");
retention.commit().await.unwrap();
let (writer, written) = writer.await.unwrap();
assert!(written.expect("the tag's writer is not aborted"), "{image}");
writer.commit().await.unwrap();

let latest: Option<i64> = sqlx::query_scalar(
"SELECT manifest_id FROM oci_tag \
WHERE repository_id = $1 AND image = $2 AND tag = 'latest'",
)
.bind(repo)
.bind(image)
.fetch_optional(&pool)
.await
.unwrap();
assert_eq!(latest, moves.then_some(other), "{image}");
let untagged: bool =
sqlx::query_scalar("SELECT untagged_since IS NOT NULL FROM oci_manifest WHERE id = $1")
.bind(tagged)
.fetch_one(&pool)
.await
.unwrap();
assert!(untagged, "{image}: the manifest `latest` left");
}
}
Loading
Loading