Experiment: Reborrow traits

compiler/rustc_borrowck/src/borrow_set.rs

@@ -2,11 +2,12 @@ use std::fmt;
 use std::ops::Index;
 
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
+use rustc_hir::Mutability;
 use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::mir::visit::{MutatingUseContext, NonUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::{self, Body, Local, Location, traversal};
-use rustc_middle::span_bug;
 use rustc_middle::ty::{RegionVid, TyCtxt};
+use rustc_middle::{bug, span_bug, ty};
 use rustc_mir_dataflow::move_paths::MoveData;
 use tracing::debug;
 
@@ -300,6 +301,71 @@ impl<'a, 'tcx> Visitor<'tcx> for GatherBorrows<'a, 'tcx> {
                 idx
             };
 
+            self.local_map.entry(borrowed_place.local).or_default().insert(idx);
+        } else if let &mir::Rvalue::Reborrow(mutability, borrowed_place) = rvalue {
+            let borrowed_place_ty = borrowed_place.ty(self.body, self.tcx).ty;
+            let &ty::Adt(reborrowed_adt, _reborrowed_args) = borrowed_place_ty.kind() else {
+                unreachable!()
+            };
+            let &ty::Adt(target_adt, assigned_args) =
+                assigned_place.ty(self.body, self.tcx).ty.kind()
+            else {
+                unreachable!()
+            };
+            let borrow = if mutability == Mutability::Mut {
+                // Reborrow
+                if target_adt.did() != reborrowed_adt.did() {
+                    bug!(
+                        "hir-typeck passed but Reborrow involves mismatching types at {location:?}"
+                    )
+                }
+                let Some(ty::GenericArgKind::Lifetime(region)) =
+                    assigned_args.get(0).map(|r| r.kind())
+                else {
+                    bug!(
+                        "hir-typeck passed but {} does not have a lifetime argument",
+                        if mutability == Mutability::Mut { "Reborrow" } else { "CoerceShared" }
+                    );
+                };
+                let region = region.as_var();
+                let kind = mir::BorrowKind::Mut { kind: mir::MutBorrowKind::Default };
+                BorrowData {
+                    kind,
+                    region,
+                    reserve_location: location,
+                    activation_location: TwoPhaseActivation::NotTwoPhase,
+                    borrowed_place,
+                    assigned_place: *assigned_place,
+                }
+            } else {
+                // CoerceShared
+                if target_adt.did() == reborrowed_adt.did() {
+                    bug!(
+                        "hir-typeck passed but CoerceShared involves matching types at {location:?}"
+                    )
+                }
+                let Some(ty::GenericArgKind::Lifetime(region)) =
+                    assigned_args.get(0).map(|r| r.kind())
+                else {
+                    bug!(
+                        "hir-typeck passed but {} does not have a lifetime argument",
+                        if mutability == Mutability::Mut { "Reborrow" } else { "CoerceShared" }
+                    );
+                };
+                let region = region.as_var();
+                let kind = mir::BorrowKind::Shared;
+                BorrowData {
+                    kind,
+                    region,
+                    reserve_location: location,
+                    activation_location: TwoPhaseActivation::NotTwoPhase,
+                    borrowed_place,
+                    assigned_place: *assigned_place,
+                }
+            };
+            let (idx, _) = self.location_map.insert_full(location, borrow);
+            let idx = BorrowIndex::from(idx);
+
             self.local_map.entry(borrowed_place.local).or_default().insert(idx);
         }
 

compiler/rustc_borrowck/src/dataflow.rs

@@ -549,7 +549,7 @@ impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Borrows<'_, 'tcx> {
     ) {
         match &stmt.kind {
             mir::StatementKind::Assign(box (lhs, rhs)) => {
-                if let mir::Rvalue::Ref(_, _, place) = rhs {
+                if let mir::Rvalue::Ref(_, _, place) | mir::Rvalue::Reborrow(_, place) = rhs {
                     if place.ignore_borrow(
                         self.tcx,
                         self.body,

compiler/rustc_borrowck/src/lib.rs

@@ -1254,6 +1254,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
         let mut error_reported = false;
 
         let borrows_in_scope = self.borrows_in_scope(location, state);
+        debug!(?borrows_in_scope, ?location);
 
         each_borrow_involving_path(
             self,
@@ -1593,6 +1594,35 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
             }
 
             Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in borrowck"),
+            &Rvalue::Reborrow(mutability, place) => {
+                let access_kind = (
+                    Deep,
+                    if mutability == Mutability::Mut {
+                        Write(WriteKind::MutableBorrow(BorrowKind::Mut {
+                            kind: MutBorrowKind::Default,
+                        }))
+                    } else {
+                        Read(ReadKind::Borrow(BorrowKind::Shared))
+                    },
+                );
+
+                self.access_place(
+                    location,
+                    (place, span),
+                    access_kind,
+                    LocalMutationIsAllowed::Yes,
+                    state,
+                );
+
+                let action = InitializationRequiringAction::Borrow;
+
+                self.check_if_path_or_subpath_is_moved(
+                    location,
+                    action,
+                    (place.as_ref(), span),
+                    state,
+                );
+            }
         }
     }
 

compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs

@@ -325,6 +325,20 @@ impl<'a, 'tcx> LoanInvalidationsGenerator<'a, 'tcx> {
             }
 
             Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in borrowck"),
+            &Rvalue::Reborrow(mutability, place) => {
+                let access_kind = (
+                    Deep,
+                    if mutability == Mutability::Mut {
+                        Reservation(WriteKind::MutableBorrow(BorrowKind::Mut {
+                            kind: MutBorrowKind::TwoPhaseBorrow,
+                        }))
+                    } else {
+                        Read(ReadKind::Borrow(BorrowKind::Shared))
+                    },
+                );
+
+                self.access_place(location, place, access_kind, LocalMutationIsAllowed::No);
+            }
         }
     }
 

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -558,6 +558,16 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
 }
 
 impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
+    fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) {
+        // check rvalue is Reborrow
+        if let Rvalue::Reborrow(mutability, rvalue) = rvalue {
+            self.add_generic_reborrow_constraint(*mutability, location, place, rvalue);
+        } else {
+            // rest of the cases
+            self.super_assign(place, rvalue, location);
+        }
+    }
+
     fn visit_span(&mut self, span: Span) {
         if !span.is_dummy() {
             debug!(?span);
@@ -635,8 +645,11 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 debug!(?rv_ty);
                 let rv_ty = self.normalize(rv_ty, location);
                 debug!("normalized rv_ty: {:?}", rv_ty);
-                if let Err(terr) =
-                    self.sub_types(rv_ty, place_ty, location.to_locations(), category)
+                // Note: we've checked Reborrow/CoerceShared type matches
+                // separately in fn visit_assign.
+                if !matches!(rv, Rvalue::Reborrow(_, _))
+                    && let Err(terr) =
+                        self.sub_types(rv_ty, place_ty, location.to_locations(), category)
                 {
                     span_mirbug!(
                         self,
@@ -1677,7 +1690,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
             | Rvalue::BinaryOp(..)
             | Rvalue::RawPtr(..)
             | Rvalue::ThreadLocalRef(..)
-            | Rvalue::Discriminant(..) => {}
+            | Rvalue::Discriminant(..)
+            | Rvalue::Reborrow(..) => {}
         }
     }
 
@@ -2243,7 +2257,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             | Rvalue::CopyForDeref(..)
             | Rvalue::UnaryOp(..)
             | Rvalue::Discriminant(..)
-            | Rvalue::WrapUnsafeBinder(..) => None,
+            | Rvalue::WrapUnsafeBinder(..)
+            | Rvalue::Reborrow(..) => None,
 
             Rvalue::Aggregate(aggregate, _) => match **aggregate {
                 AggregateKind::Adt(_, _, _, user_ty, _) => user_ty,
@@ -2441,6 +2456,120 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         }
     }
 
+    fn add_generic_reborrow_constraint(
+        &mut self,
+        mutability: Mutability,
+        location: Location,
+        dest: &Place<'tcx>,
+        borrowed_place: &Place<'tcx>,
+    ) {
+        // In Polonius mode, we also push a `loan_issued_at` fact
+        // linking the loan to the region (in some cases, though,
+        // there is no loan associated with this borrow expression --
+        // that occurs when we are borrowing an unsafe place, for
+        // example).
+        // if let Some(polonius_facts) = polonius_facts {
+        //     let _prof_timer = self.infcx.tcx.prof.generic_activity("polonius_fact_generation");
+        //     if let Some(borrow_index) = borrow_set.get_index_of(&location) {
+        //         let region_vid = borrow_region.as_var();
+        //         polonius_facts.loan_issued_at.push((
+        //             region_vid.into(),
+        //             borrow_index,
+        //             location_table.mid_index(location),
+        //         ));
+        //     }
+        // }
+
+        // If we are reborrowing the referent of another reference, we
+        // need to add outlives relationships. In a case like `&mut
+        // *p`, where the `p` has type `&'b mut Foo`, for example, we
+        // need to ensure that `'b: 'a`.
+
+        debug!(
+            "add_generic_reborrow_constraint({:?}, {:?}, {:?}, {:?})",
+            mutability, location, dest, borrowed_place
+        );
+
+        let tcx = self.infcx.tcx;
+        let def = self.body.source.def_id().expect_local();
+        let upvars = tcx.closure_captures(def);
+        let field =
+            path_utils::is_upvar_field_projection(tcx, upvars, borrowed_place.as_ref(), self.body);
+        let category = if let Some(field) = field {
+            ConstraintCategory::ClosureUpvar(field)
+        } else {
+            ConstraintCategory::Boring
+        };
+
+        let dest_ty = dest.ty(self.body, tcx).ty;
+        let borrowed_ty = borrowed_place.ty(self.body, tcx).ty;
+        let ty::Adt(_, args) = dest_ty.kind() else { bug!() };
+        let [arg, ..] = ***args else { bug!() };
+        let ty::GenericArgKind::Lifetime(reborrow_region) = arg.kind() else { bug!() };
+        self.constraints.liveness_constraints.add_location(reborrow_region.as_var(), location);
+
+        if mutability.is_not() {
+            // FIXME: for shared reborrow we need to relate the types manually,
+            // field by field with CoerceShared drilling down and down and down.
+            // We cannot just attempt to relate T and <T as CoerceShared>::Target
+            // by calling relate_types.
+            let ty::Adt(dest_adt, dest_args) = dest_ty.kind() else { unreachable!() };
+            let ty::Adt(borrowed_adt, borrowed_args) = borrowed_ty.kind() else { unreachable!() };
+            let borrowed_fields = borrowed_adt.all_fields().collect::<Vec<_>>();
+            for dest_field in dest_adt.all_fields() {
+                let Some(borrowed_field) =
+                    borrowed_fields.iter().find(|f| f.name == dest_field.name)
+                else {
+                    continue;
+                };
+                let dest_ty = dest_field.ty(tcx, dest_args);
+                let borrowed_ty = borrowed_field.ty(tcx, borrowed_args);
+                if let (
+                    ty::Ref(borrow_region, _, Mutability::Mut),
+                    ty::Ref(ref_region, _, Mutability::Not),
+                ) = (borrowed_ty.kind(), dest_ty.kind())
+                {
+                    self.relate_types(
+                        borrowed_ty.peel_refs(),
+                        ty::Variance::Covariant,
+                        dest_ty.peel_refs(),
+                        location.to_locations(),
+                        category,
+                    )
+                    .unwrap();
+                    self.constraints.outlives_constraints.push(OutlivesConstraint {
+                        sup: ref_region.as_var(),
+                        sub: borrow_region.as_var(),
+                        locations: location.to_locations(),
+                        span: location.to_locations().span(self.body),
+                        category,
+                        variance_info: ty::VarianceDiagInfo::default(),
+                        from_closure: false,
+                    });
+                } else {
+                    self.relate_types(
+                        borrowed_ty,
+                        ty::Variance::Covariant,
+                        dest_ty,
+                        location.to_locations(),
+                        category,
+                    )
+                    .unwrap();
+                }
+            }
+        } else {
+            // Exclusive reborrow
+            self.relate_types(
+                borrowed_ty,
+                ty::Variance::Covariant,
+                dest_ty,
+                location.to_locations(),
+                category,
+            )
+            .unwrap();
+        }
+    }
+
     fn prove_aggregate_predicates(
         &mut self,
         aggregate_kind: &AggregateKind<'tcx>,

compiler/rustc_codegen_cranelift/src/base.rs

@@ -624,6 +624,11 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
                     let val = codegen_operand(fx, operand);
                     lval.write_cvalue(fx, val);
                 }
+                Rvalue::Reborrow(_, place) => {
+                    let cplace = codegen_place(fx, place);
+                    let val = cplace.to_cvalue(fx);
+                    lval.write_cvalue(fx, val)
+                }
                 Rvalue::Ref(_, _, place) | Rvalue::RawPtr(_, place) => {
                     let place = codegen_place(fx, place);
                     let ref_ = place.place_ref(fx, lval.layout());

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -709,6 +709,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let layout = bx.cx().layout_of(binder_ty);
                 OperandRef { val: operand.val, layout, move_annotation: None }
             }
+            // Note: Exclusive reborrowing is always equal to a memcpy, as the
+            // types do not change. Generic shared reborrowing is not
+            // (necessarily) a simple memcpy, but currently the coherence check
+            // places such restrictions on the CoerceShared trait as to
+            // guarantee that it is.
+            mir::Rvalue::Reborrow(_, place) => self.codegen_operand(bx, &mir::Operand::Copy(place)),
             mir::Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in codegen"),
             mir::Rvalue::ShallowInitBox(..) => bug!("`ShallowInitBox` in codegen"),
         }

compiler/rustc_const_eval/src/check_consts/check.rs

@@ -699,6 +699,8 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             Rvalue::WrapUnsafeBinder(..) => {
                 // Unsafe binders are always trivial to create.
             }
+
+            Rvalue::Reborrow(..) => {}
         }
     }
 

compiler/rustc_const_eval/src/check_consts/qualifs.rs

@@ -276,6 +276,8 @@ where
             // Otherwise, proceed structurally...
             operands.iter().any(|o| in_operand::<Q, _>(cx, in_local, o))
         }
+
+        Rvalue::Reborrow(_, place) => in_place::<Q, _>(cx, in_local, place.as_ref()),
     }
 }
 

compiler/rustc_const_eval/src/check_consts/resolver.rs

@@ -191,6 +191,16 @@ where
                 }
             }
 
+            mir::Rvalue::Reborrow(_, borrowed_place) => {
+                if !borrowed_place.is_indirect() {
+                    let place_ty = borrowed_place.ty(self.ccx.body, self.ccx.tcx).ty;
+                    if Q::in_any_value_of_ty(self.ccx, place_ty) {
+                        self.state.qualif.insert(borrowed_place.local);
+                        self.state.borrow.insert(borrowed_place.local);
+                    }
+                }
+            }
+
             mir::Rvalue::Cast(..)
             | mir::Rvalue::ShallowInitBox(..)
             | mir::Rvalue::Use(..)