[Calico/VPP NSM integration] Forwarder should create named net NS instead of using /proc/1/fd/x

[Bolodya1997]

Parent issue

networkservicemesh/integration-k8s-kind#325

Issue

If VPP forwarder and VPP are located in different containers, they cannot share /proc/1/fd/x.
Forwarder performs API requests to VPP with file describing Client/Endpoint net NS, currently it is /proc/1/fd/x received from recvfd - it is not accessible from VPP container and such request fails.

Possible solution

Add netns chain elements creating named net NS and replacing netNSURL in kernel mechanism before Requesting/Closing kernel chain element.

common.go

func toNamed(ctx context.Context, nsURL string) (string, error) {
	u, err := url.Parse(nsURL)
	if err != nil {
		return "", err
	}
	nsPath := u.Path

	switch {
	case strings.HasPrefix(nsPath, namedPath):
		return nsURL, nil
	case !strings.HasPrefix(nsPath, unnamedPath):
		return "", errors.Errorf("[%s] should be located under: %s", nsPath, unnamedPath)
	}

	u.Path = path.Join(namedPath, fmt.Sprintf("ns-%s", path.Base(nsPath)))

	switch f, err := os.OpenFile(u.Path, os.O_CREATE|os.O_EXCL, 0444); {
	case os.IsExist(err):
		return u.String(), nil
	case err != nil:
		return "", err
	default:
		_ = f.Close()
	}

	if err := unix.Mount(nsPath, u.Path, "bind", syscall.MS_BIND, ""); err != nil {
		return "", err
	}
	if ctx != nil {
		go func() {
			<-ctx.Done()
			_ = unix.Unmount(u.Path, 0)
			_ = os.RemoveAll(u.Path)
		}()
	}

	return u.String(), nil
}

func fromNamed(namedNSURL string) string {
	u, _ := url.Parse(namedNSURL)
	namedNSPath := u.Path

	if strings.HasPrefix(namedNSPath, unnamedPath) {
		return namedNSURL
	}

	u.Path = path.Join(unnamedPath, strings.TrimPrefix(namedNSPath, path.Join(namedPath, "ns-")))

	return u.String()
}

server.go

func (s *netnsServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (conn *networkservice.Connection, err error) {
	if mech := kernel.ToMechanism(request.GetConnection().GetMechanism()); mech != nil {
		nsURL := mech.GetNetNSURL()

		namedNSURL, err := toNamed(nil, mech.GetNetNSURL())
		if err != nil {
			return nil, err
		}

		mech.SetNetNSURL(namedNSURL)
		defer func() {
			if conn != nil {
				kernel.ToMechanism(conn.GetMechanism()).SetNetNSURL(nsURL)
			}
		}()
	}
	return next.Server(ctx).Request(ctx, request)
}

func (s *netnsServer) Close(ctx context.Context, conn *networkservice.Connection) (*emptypb.Empty, error) {
	if mech := kernel.ToMechanism(conn.GetMechanism()); mech != nil {
		nsURL := mech.GetNetNSURL()

		namedNSURL, err := toNamed(ctx, mech.GetNetNSURL())
		if err != nil {
			return nil, err
		}

		mech.SetNetNSURL(namedNSURL)
		defer func() {
			mech.SetNetNSURL(nsURL)
		}()
	}
	return next.Server(ctx).Close(ctx, conn)
}

client.go

func (s *netnsClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) {
	if mech := kernel.ToMechanism(request.GetConnection().GetMechanism()); mech != nil {
		mech.SetNetNSURL(fromNamed(mech.GetNetNSURL()))
	}

	conn, err := next.Client(ctx).Request(ctx, request, opts...)
	if err != nil {
		return nil, err
	}

	if mech := kernel.ToMechanism(conn.GetMechanism()); mech != nil {
		namedNSURL, err := toNamed(nil, mech.GetNetNSURL())
		if err != nil {
			return nil, err // FIXME: postpone
		}

		mech.SetNetNSURL(namedNSURL)
	}

	return conn, err
}

func (s *netnsClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*emptypb.Empty, error) {
	if mech := kernel.ToMechanism(conn.GetMechanism()); mech != nil {
		mech.SetNetNSURL(fromNamed(mech.GetNetNSURL()))
	}

	rv, err := next.Client(ctx).Close(ctx, conn, opts...)

	if mech := kernel.ToMechanism(conn.GetMechanism()); mech != nil {
		namedNSURL, err := toNamed(ctx, mech.GetNetNSURL())
		if err != nil {
			return nil, err
		}

		mech.SetNetNSURL(namedNSURL)
	}

	return rv, err
}

[Bolodya1997]

@edwarnicke @denis-tingaikin
cc

[edwarnicke]

@Bolodya1997 For Calico/VPP integration, run in the host:pid space for the forwarder. The VPP plugin is also running in host:pid for Calico/VPP, and Calico/VPP itself passes to VPP things in /proc/${pid}/... so we know this works in host:pid