fix(xray): fall back to the userspace TUN where kernel TUN cannot be set up

The core puts a WireGuard outbound on a kernel TUN whenever it holds
CAP_NET_ADMIN and noKernelTun is unset, the default for a hand-added outbound.
That needs a writable /proc/sys/net and /dev/net/tun, and where either is
missing the core does not fall back: it aborts the whole start. The shipped
docker-compose grants NET_ADMIN for fail2ban while Docker keeps /proc/sys
read-only and exposes no /dev/net/tun, so one such outbound took every inbound
down with the #6796 error; any unit or drop-in that keeps /proc/sys/net
read-only did the same.

xray.KernelTunBlocked probes once, from the panel process whose capabilities
and mounts the core inherits, whether the core would pick a kernel TUN it
cannot create. GetXrayConfig then sets noKernelTun on every WireGuard outbound,
so cold starts and hot reloads both get it. Hosts that can create the device
keep the kernel TUN.

Verified with the pinned core's "run -test" on the generated config, in a
NET_ADMIN container as docker-compose runs it and under systemd 257: the
container and the v3.10.0 unit report blocked and pass on gVisor, the container
fails with the #6796 error when the GetXrayConfig hook is bypassed, and the new
unit keeps the kernel TUN.

Refs #6796

internal/web/service/xray.go

@@ -189,6 +189,9 @@ func (s *XrayService) GetXrayConfig() (*xray.Config, error) {
 	if err := transformAmneziaWGOutbounds(xrayConfig); err != nil {
 		return nil, err
 	}
+	if xray.KernelTunBlocked() {
+		xrayConfig.OutboundConfigs = forceUserspaceWireGuardTun(xrayConfig.OutboundConfigs)
+	}
 
 	_, _, _ = s.inboundService.AddTraffic(nil, nil)
 
@@ -1716,3 +1719,38 @@ func upgradeStreamLegacyUDPHop(stream map[string]any) bool {
 func normalizeStreamUDPHop(stream map[string]any) bool {
 	return maskcompat.NormalizeUDPHop(stream["finalmask"])
 }
+
+// forceUserspaceWireGuardTun moves every WireGuard outbound to the gVisor TUN (#6796). Values
+// stay raw so the template's other keys reach the core byte for byte.
+func forceUserspaceWireGuardTun(raw json_util.RawMessage) json_util.RawMessage {
+	var outbounds []map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &outbounds); err != nil {
+		return raw
+	}
+	for _, ob := range outbounds {
+		var protocol, tag string
+		if json.Unmarshal(ob["protocol"], &protocol) != nil || !strings.EqualFold(protocol, "wireguard") {
+			continue
+		}
+		var settings map[string]json.RawMessage
+		if json.Unmarshal(ob["settings"], &settings) != nil || settings == nil {
+			continue
+		}
+		wasUserspace := string(settings["noKernelTun"]) == "true"
+		settings["noKernelTun"] = json.RawMessage("true")
+		rewritten, err := json.Marshal(settings)
+		if err != nil {
+			continue
+		}
+		ob["settings"] = rewritten
+		if !wasUserspace {
+			_ = json.Unmarshal(ob["tag"], &tag)
+			logger.Warningf("Outbound %q: kernel TUN cannot be set up here, running WireGuard on the userspace TUN instead (#6796)", tag)
+		}
+	}
+	rewritten, err := json.Marshal(outbounds)
+	if err != nil {
+		return raw
+	}
+	return rewritten
+}

internal/web/service/xray_wireguard_config_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/mhsanaei/3x-ui/v3/internal/database"
 	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
+	"github.com/mhsanaei/3x-ui/v3/internal/util/json_util"
 )
 
 func wgTestSecretKey() string {
@@ -270,3 +271,35 @@ func TestGetXrayConfigWireGuardNoClientsEmitsEmptyPeers(t *testing.T) {
 		t.Fatalf("expected empty peers, got %v", peers)
 	}
 }
+
+// #6796: where the core would pick a kernel TUN it cannot set up (Docker, a sandboxed unit),
+// one WireGuard outbound aborts the whole start unless it is moved to the userspace TUN.
+func TestForceUserspaceWireGuardTun(t *testing.T) {
+	cases := []struct {
+		name, protocol, settings string
+	}{
+		{"kernel TUN requested", "wireguard", `{"noKernelTun":false,"mtu":1420}`},
+		{"kernel TUN by default, protocol cased like the core accepts", "WireGuard", `{"mtu":1420}`},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			raw := json_util.RawMessage(`[{"protocol":"freedom","tag":"direct","settings":{}},` +
+				`{"protocol":"` + tc.protocol + `","tag":"warp","settings":` + tc.settings + `}]`)
+			var outbounds []struct {
+				Settings map[string]any `json:"settings"`
+			}
+			if err := json.Unmarshal(forceUserspaceWireGuardTun(raw), &outbounds); err != nil {
+				t.Fatal(err)
+			}
+			if got := outbounds[1].Settings["noKernelTun"]; got != true {
+				t.Fatalf("wireguard noKernelTun = %v, want true", got)
+			}
+			if got := outbounds[1].Settings["mtu"]; got != float64(1420) {
+				t.Fatalf("wireguard mtu = %v, want the template's 1420 kept", got)
+			}
+			if _, ok := outbounds[0].Settings["noKernelTun"]; ok {
+				t.Fatalf("freedom outbound gained noKernelTun: %v", outbounds[0].Settings)
+			}
+		})
+	}
+}

internal/xray/kernel_tun_linux.go

@@ -0,0 +1,52 @@
+package xray
+
+import (
+	"errors"
+	"io/fs"
+	"os"
+	"sync"
+
+	"golang.org/x/sys/unix"
+)
+
+// The files the core's WireGuard kernel TUN opens before it creates the device
+// (proxy/wireguard/tun_linux.go); a missing one is skipped there too.
+var kernelTunSysctls = []string{
+	"/proc/sys/net/ipv4/conf/all/rp_filter",
+	"/proc/sys/net/ipv6/conf/all/disable_ipv6",
+	"/proc/sys/net/ipv6/conf/all/rp_filter",
+}
+
+// KernelTunBlocked reports that the core would put a WireGuard outbound on a kernel TUN
+// (it holds CAP_NET_ADMIN) where a read-only /proc/sys or a missing /dev/net/tun aborts its start.
+var KernelTunBlocked = sync.OnceValue(func() bool {
+	return hasEffectiveNetAdmin() && !kernelTunReachable()
+})
+
+func hasEffectiveNetAdmin() bool {
+	hdr := unix.CapUserHeader{Version: unix.LINUX_CAPABILITY_VERSION_3}
+	var data [2]unix.CapUserData
+	if err := unix.Capget(&hdr, &data[0]); err != nil {
+		return false
+	}
+	return data[0].Effective&(1<<unix.CAP_NET_ADMIN) != 0
+}
+
+func kernelTunReachable() bool {
+	for _, path := range kernelTunSysctls {
+		f, err := os.OpenFile(path, os.O_WRONLY, 0)
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		}
+		if err != nil {
+			return false
+		}
+		_ = f.Close()
+	}
+	tun, err := os.OpenFile("/dev/net/tun", os.O_RDWR, 0)
+	if err != nil {
+		return false
+	}
+	_ = tun.Close()
+	return true
+}

internal/xray/kernel_tun_other.go

@@ -0,0 +1,6 @@
+//go:build !linux
+
+package xray
+
+// KernelTunBlocked is always false off Linux: the core never puts WireGuard on a kernel TUN there.
+func KernelTunBlocked() bool { return false }