Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions extractor/src/discogs_downloader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,16 @@ use crate::types::{LocalFileInfo, S3FileInfo};
// S3 file names already contain the full key (e.g., "data/2026/discogs_...xml.gz")
// so no prefix stripping or re-prepending is needed.
const DISCOGS_DATA_URL: &str = "https://data.discogs.com/";
// Discogs has been observed returning Retry-After ~36 minutes; with 5 attempts
// we give ourselves enough headroom to ride out a single bad cooldown without
// the docker restart loop sliding the limiter window forward.
const MAX_DOWNLOAD_RETRIES: u32 = 5;
// This retry loop only fires for *post-connect* failures — partial reads,
// flush/sync errors, transport drops mid-stream. Rate-limit (HTTP 429 / 503)
// handling lives upstream in `polite_http::PoliteClient` and never reaches
// this loop, so 3 attempts at 2s base is plenty to ride out a brief network
// hiccup. Higher values bloat CI runtime — integration tests in `tests/`
// see `cfg(not(test))` and pay the full backoff per failing-download test.
const MAX_DOWNLOAD_RETRIES: u32 = 3;

#[cfg(not(test))]
const RETRY_BASE_DELAY_MS: u64 = 30_000;
const RETRY_BASE_DELAY_MS: u64 = 2_000;
#[cfg(test)]
const RETRY_BASE_DELAY_MS: u64 = 10;

Expand Down
6 changes: 4 additions & 2 deletions extractor/src/musicbrainz_downloader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -143,11 +143,13 @@ pub fn find_latest_mb_directory(root: &Path) -> Option<PathBuf> {
const MB_ENTITIES: &[&str] = &["artist", "label", "release-group", "release"];

#[allow(dead_code)]
const MB_MAX_DOWNLOAD_RETRIES: u32 = 5;
const MB_MAX_DOWNLOAD_RETRIES: u32 = 3;

// Post-connect transport-error retry — see the equivalent comment in
// `discogs_downloader.rs`. Rate-limit handling lives in `polite_http`.
#[cfg(not(test))]
#[allow(dead_code)]
const MB_RETRY_BASE_DELAY_MS: u64 = 30_000;
const MB_RETRY_BASE_DELAY_MS: u64 = 2_000;
#[cfg(test)]
#[allow(dead_code)]
const MB_RETRY_BASE_DELAY_MS: u64 = 10;
Expand Down
Loading