blob: 81d0745818c5912dcb291da73e768e5f41f309fb [file] [log] [blame]
use std::borrow::Cow;
use gix_date::SecondsSinceUnixEpoch;
use gix_negotiate::Flags;
use gix_odb::HeaderExt;
use gix_pack::Find;
use crate::remote::{fetch, fetch::Shallow};
type Queue = gix_revision::PriorityQueue<SecondsSinceUnixEpoch, gix_hash::ObjectId>;
/// The error returned during negotiation.
#[derive(Debug, thiserror::Error)]
#[allow(missing_docs)]
pub enum Error {
#[error("We were unable to figure out what objects the server should send after {rounds} round(s)")]
NegotiationFailed { rounds: usize },
#[error(transparent)]
LookupCommitInGraph(#[from] gix_revision::graph::lookup::commit::Error),
#[error(transparent)]
InitRefsIterator(#[from] crate::reference::iter::init::Error),
#[error(transparent)]
InitRefsIteratorPlatform(#[from] crate::reference::iter::Error),
#[error(transparent)]
ObtainRefDuringIteration(#[from] Box<dyn std::error::Error + Send + Sync + 'static>),
#[error(transparent)]
LoadIndex(#[from] gix_odb::store::load_index::Error),
}
#[must_use]
pub(crate) enum Action {
/// None of the remote refs moved compared to our last recorded state (via tracking refs), so there is nothing to do at all,
/// not even a ref update.
NoChange,
/// Don't negotiate, don't fetch the pack, skip right to updating the references.
///
/// This happens if we already have all local objects even though the server seems to have changed.
SkipToRefUpdate,
/// We can't know for sure if fetching *is not* needed, so we go ahead and negotiate.
MustNegotiate {
/// Each `ref_map.mapping` has a slot here which is `true` if we have the object the remote ref points to locally.
remote_ref_target_known: Vec<bool>,
},
}
/// This function is modeled after the similarly named one in the git codebase to do the following:
///
/// * figure out all advertised refs on the remote *that we already have* and keep track of the oldest one as cutoff date.
/// * mark all of our own refs as tips for a traversal.
/// * mark all their parents, recursively, up to (and including) the cutoff date up to which we have seen the servers commit that we have.
/// * pass all known-to-be-common-with-remote commits to the negotiator as common commits.
///
/// This is done so that we already find the most recent common commits, even if we are ahead, which is *potentially* better than
/// what we would get if we would rely on tracking refs alone, particularly if one wouldn't trust the tracking refs for some reason.
///
/// Note that git doesn't trust its own tracking refs as the server *might* have changed completely, for instance by force-pushing, so
/// marking our local tracking refs as known is something that's actually not proven to be correct so it's not done.
///
/// Additionally, it does what's done in `transport.c` and we check if a fetch is actually needed as at least one advertised ref changed.
///
/// Finally, we also mark tips in the `negotiator` in one go to avoid traversing all refs twice, since we naturally encounter all tips during
/// our own walk.
///
/// Return whether or not we should negotiate, along with a queue for later use.
pub(crate) fn mark_complete_and_common_ref(
repo: &crate::Repository,
negotiator: &mut dyn gix_negotiate::Negotiator,
graph: &mut gix_negotiate::Graph<'_>,
ref_map: &fetch::RefMap,
shallow: &fetch::Shallow,
mapping_is_ignored: impl Fn(&fetch::Mapping) -> bool,
) -> Result<Action, Error> {
let _span = gix_trace::detail!("mark_complete_and_common_ref", mappings = ref_map.mappings.len());
if let fetch::Shallow::Deepen(0) = shallow {
// Avoid deepening (relative) with zero as it seems to upset the server. Git also doesn't actually
// perform the negotiation for some reason (couldn't find it in code).
return Ok(Action::NoChange);
}
if let Some(fetch::Mapping {
remote: fetch::Source::Ref(gix_protocol::handshake::Ref::Unborn { .. }),
..
}) = ref_map.mappings.last().filter(|_| ref_map.mappings.len() == 1)
{
// There is only an unborn branch, as the remote has an empty repository. This means there is nothing to do except for
// possibly reproducing the unborn branch locally.
return Ok(Action::SkipToRefUpdate);
}
// Compute the cut-off date by checking which of the refs advertised (and matched in refspecs) by the remote we have,
// and keep the oldest one.
let mut cutoff_date = None::<SecondsSinceUnixEpoch>;
let mut num_mappings_with_change = 0;
let mut remote_ref_target_known: Vec<bool> = std::iter::repeat(false).take(ref_map.mappings.len()).collect();
let mut remote_ref_included: Vec<bool> = std::iter::repeat(false).take(ref_map.mappings.len()).collect();
for (mapping_idx, mapping) in ref_map.mappings.iter().enumerate() {
let want_id = mapping.remote.as_id();
let have_id = mapping.local.as_ref().and_then(|name| {
// this is the only time git uses the peer-id.
let r = repo.find_reference(name).ok()?;
r.target().try_id().map(ToOwned::to_owned)
});
// Even for ignored mappings we want to know if the `want` is already present locally, so skip nothing else.
if !mapping_is_ignored(mapping) {
remote_ref_included[mapping_idx] = true;
// Like git, we don't let known unchanged mappings participate in the tree traversal
if want_id.zip(have_id).map_or(true, |(want, have)| want != have) {
num_mappings_with_change += 1;
}
}
if let Some(commit) = want_id
.and_then(|id| graph.try_lookup_or_insert_commit(id.into(), |_| {}).transpose())
.transpose()?
{
remote_ref_target_known[mapping_idx] = true;
cutoff_date = cutoff_date.unwrap_or_default().max(commit.commit_time).into();
} else if want_id.map_or(false, |maybe_annotated_tag| repo.objects.contains(maybe_annotated_tag)) {
remote_ref_target_known[mapping_idx] = true;
}
}
if matches!(shallow, Shallow::NoChange) {
if num_mappings_with_change == 0 {
return Ok(Action::NoChange);
} else if remote_ref_target_known
.iter()
.zip(remote_ref_included)
.filter_map(|(known, included)| included.then_some(known))
.all(|known| *known)
{
return Ok(Action::SkipToRefUpdate);
}
}
// color our commits as complete as identified by references, unconditionally
// (`git` is conditional here based on `deepen`, but it doesn't make sense and it's hard to extract from history when that happened).
let mut queue = Queue::new();
mark_all_refs_in_repo(repo, graph, &mut queue, Flags::COMPLETE)?;
mark_alternate_complete(repo, graph, &mut queue)?;
// Keep track of the tips, which happen to be on our queue right, before we traverse the graph with cutoff.
let tips = if let Some(cutoff) = cutoff_date {
let tips = Cow::Owned(queue.clone());
// color all their parents up to the cutoff date, the oldest commit we know the server has.
mark_recent_complete_commits(&mut queue, graph, cutoff)?;
tips
} else {
Cow::Borrowed(&queue)
};
gix_trace::detail!("mark known_common").into_scope(|| -> Result<_, Error> {
// mark all complete advertised refs as common refs.
for mapping in ref_map
.mappings
.iter()
.zip(remote_ref_target_known.iter().copied())
// We need this filter as the graph wouldn't contain annotated tags.
.filter_map(|(mapping, known)| (!known).then_some(mapping))
{
let want_id = mapping.remote.as_id();
if let Some(common_id) = want_id
.and_then(|id| graph.get(id).map(|c| (c, id)))
.filter(|(c, _)| c.data.flags.contains(Flags::COMPLETE))
.map(|(_, id)| id)
{
negotiator.known_common(common_id.into(), graph)?;
}
}
Ok(())
})?;
// As negotiators currently may rely on getting `known_common` calls first and tips after, we adhere to that which is the only
// reason we cached the set of tips.
gix_trace::detail!("mark tips", num_tips = tips.len()).into_scope(|| -> Result<_, Error> {
for tip in tips.iter_unordered() {
negotiator.add_tip(*tip, graph)?;
}
Ok(())
})?;
Ok(Action::MustNegotiate {
remote_ref_target_known,
})
}
/// Create a predicate that checks if a refspec mapping should be ignored.
///
/// We want to ignore mappings during negotiation if they would be handled implicitly by the server, which is the case
/// when tags would be sent implicitly due to `Tags::Included`.
pub(crate) fn make_refmapping_ignore_predicate(
fetch_tags: fetch::Tags,
ref_map: &fetch::RefMap,
) -> impl Fn(&fetch::Mapping) -> bool + '_ {
// With included tags, we have to keep mappings of tags to handle them later when updating refs, but we don't want to
// explicitly `want` them as the server will determine by itself which tags are pointing to a commit it wants to send.
// If we would not exclude implicit tag mappings like this, we would get too much of the graph.
let tag_refspec_to_ignore = matches!(fetch_tags, crate::remote::fetch::Tags::Included)
.then(|| fetch_tags.to_refspec())
.flatten();
move |mapping| {
tag_refspec_to_ignore.map_or(false, |tag_spec| {
mapping
.spec_index
.implicit_index()
.and_then(|idx| ref_map.extra_refspecs.get(idx))
.map_or(false, |spec| spec.to_ref() == tag_spec)
})
}
}
/// Add all `wants` to `arguments`, which is the unpeeled direct target that the advertised remote ref points to.
pub(crate) fn add_wants(
repo: &crate::Repository,
arguments: &mut gix_protocol::fetch::Arguments,
ref_map: &fetch::RefMap,
mapping_known: &[bool],
shallow: &fetch::Shallow,
mapping_is_ignored: impl Fn(&fetch::Mapping) -> bool,
) {
// When using shallow, we can't exclude `wants` as the remote won't send anything then. Thus we have to resend everything
// we have as want instead to get exactly the same graph, but possibly deepened.
let is_shallow = !matches!(shallow, fetch::Shallow::NoChange);
let wants = ref_map
.mappings
.iter()
.zip(mapping_known)
.filter_map(|(m, known)| (is_shallow || !*known).then_some(m))
.filter(|m| !mapping_is_ignored(m));
for want in wants {
let id_on_remote = want.remote.as_id();
if !arguments.can_use_ref_in_want() || matches!(want.remote, fetch::Source::ObjectId(_)) {
if let Some(id) = id_on_remote {
arguments.want(id);
}
} else {
arguments.want_ref(
want.remote
.as_name()
.expect("name available if this isn't an object id"),
)
}
let id_is_annotated_tag_we_have = id_on_remote
.and_then(|id| repo.objects.header(id).ok().map(|h| (id, h)))
.filter(|(_, h)| h.kind() == gix_object::Kind::Tag)
.map(|(id, _)| id);
if let Some(tag_on_remote) = id_is_annotated_tag_we_have {
// Annotated tags are not handled at all by negotiators in the commit-graph - they only see commits and thus won't
// ever add `have`s for tags. To correct for that, we add these haves here to avoid getting them sent again.
arguments.have(tag_on_remote)
}
}
}
/// Remove all commits that are more recent than the cut-off, which is the commit time of the oldest common commit we have with the server.
fn mark_recent_complete_commits(
queue: &mut Queue,
graph: &mut gix_negotiate::Graph<'_>,
cutoff: SecondsSinceUnixEpoch,
) -> Result<(), Error> {
let _span = gix_trace::detail!("mark_recent_complete", queue_len = queue.len());
while let Some(id) = queue
.peek()
.and_then(|(commit_time, id)| (commit_time >= &cutoff).then_some(*id))
{
queue.pop_value();
let commit = graph.get(&id).expect("definitely set when adding tips or parents");
for parent_id in commit.parents.clone() {
let mut was_complete = false;
if let Some(parent) = graph
.try_lookup_or_insert_commit(parent_id, |md| {
was_complete = md.flags.contains(Flags::COMPLETE);
md.flags |= Flags::COMPLETE
})?
.filter(|_| !was_complete)
{
queue.insert(parent.commit_time, parent_id)
}
}
}
Ok(())
}
fn mark_all_refs_in_repo(
repo: &crate::Repository,
graph: &mut gix_negotiate::Graph<'_>,
queue: &mut Queue,
mark: Flags,
) -> Result<(), Error> {
let _span = gix_trace::detail!("mark_all_refs");
for local_ref in repo.references()?.all()?.peeled() {
let local_ref = local_ref?;
let id = local_ref.id().detach();
let mut is_complete = false;
if let Some(commit) = graph
.try_lookup_or_insert_commit(id, |md| {
is_complete = md.flags.contains(Flags::COMPLETE);
md.flags |= mark
})?
.filter(|_| !is_complete)
{
queue.insert(commit.commit_time, id);
};
}
Ok(())
}
fn mark_alternate_complete(
repo: &crate::Repository,
graph: &mut gix_negotiate::Graph<'_>,
queue: &mut Queue,
) -> Result<(), Error> {
let alternates = repo.objects.store_ref().alternate_db_paths()?;
let _span = gix_trace::detail!("mark_alternate_refs", num_odb = alternates.len());
for alternate_repo in alternates.into_iter().filter_map(|path| {
path.ancestors()
.nth(1)
.and_then(|git_dir| crate::open_opts(git_dir, repo.options.clone()).ok())
}) {
mark_all_refs_in_repo(&alternate_repo, graph, queue, Flags::ALTERNATE | Flags::COMPLETE)?;
}
Ok(())
}
/// Negotiate the nth `round` with `negotiator` sending `haves_to_send` after possibly making the known common commits
/// as sent by the remote known to `negotiator` using `previous_response` if this isn't the first round.
/// All `haves` are added to `arguments` accordingly.
/// Returns the amount of haves actually sent.
pub(crate) fn one_round(
negotiator: &mut dyn gix_negotiate::Negotiator,
graph: &mut gix_negotiate::Graph<'_>,
haves_to_send: usize,
arguments: &mut gix_protocol::fetch::Arguments,
previous_response: Option<&gix_protocol::fetch::Response>,
mut common: Option<&mut Vec<gix_hash::ObjectId>>,
) -> Result<(usize, bool), Error> {
let mut seen_ack = false;
if let Some(response) = previous_response {
use gix_protocol::fetch::response::Acknowledgement;
for ack in response.acknowledgements() {
match ack {
Acknowledgement::Common(id) => {
seen_ack = true;
negotiator.in_common_with_remote(*id, graph)?;
if let Some(ref mut common) = common {
common.push(*id);
}
}
Acknowledgement::Ready => {
// NOTE: In git, there is some logic dealing with whether to expect a DELIM or FLUSH package,
// but we handle this with peeking.
}
Acknowledgement::Nak => {}
}
}
}
// `common` is set only if this is a stateless transport, and we repeat previously confirmed common commits as HAVE, because
// we are not going to repeat them otherwise.
if let Some(common) = common {
for have_id in common {
arguments.have(have_id);
}
}
let mut haves_sent = 0;
for have_id in (0..haves_to_send).map_while(|_| negotiator.next_have(graph)) {
arguments.have(have_id?);
haves_sent += 1;
}
// Note that we are differing from the git implementation, which does an extra-round of with no new haves sent at all.
// For us it seems better to just say we are done when we know we are done, as potentially additional acks won't affect the
// queue of any of our implementation at all (so the negotiator won't come up with more haves next time either).
Ok((haves_sent, seen_ack))
}