inbound_node.go 52 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564
  1. package service
  2. import (
  3. "context"
  4. "errors"
  5. "fmt"
  6. "sort"
  7. "strings"
  8. "sync"
  9. "time"
  10. "github.com/mhsanaei/3x-ui/v3/internal/database"
  11. "github.com/mhsanaei/3x-ui/v3/internal/database/model"
  12. "github.com/mhsanaei/3x-ui/v3/internal/logger"
  13. "github.com/mhsanaei/3x-ui/v3/internal/web/runtime"
  14. "github.com/mhsanaei/3x-ui/v3/internal/xray"
  15. "gorm.io/gorm"
  16. "gorm.io/gorm/clause"
  17. )
  18. var reportedRemoteTagConflict sync.Map
  19. // nodeBulkPushThreshold caps how many per-client RPCs a single operation will
  20. // stream to a remote node. Above it, the panel marks the node dirty instead and
  21. // lets one ReconcileNode push converge the whole inbound — far cheaper than M
  22. // sequential round-trips. Small ops stay on the live per-client path.
  23. const nodeBulkPushThreshold = 32
  24. func (s *InboundService) runtimeFor(ib *model.Inbound) (runtime.Runtime, error) {
  25. mgr := runtime.GetManager()
  26. if mgr == nil {
  27. return nil, fmt.Errorf("runtime manager not initialised")
  28. }
  29. return mgr.RuntimeFor(ib.NodeID)
  30. }
  31. func (s *InboundService) nodePushPlan(ib *model.Inbound) (runtime.Runtime, bool, bool, error) {
  32. if ib.NodeID == nil {
  33. rt, err := s.runtimeFor(ib)
  34. if err != nil {
  35. return nil, false, false, nil
  36. }
  37. return rt, true, false, nil
  38. }
  39. nodeSvc := NodeService{}
  40. enabled, status, _, _, err := nodeSvc.NodeSyncState(*ib.NodeID)
  41. if err != nil {
  42. return nil, false, false, err
  43. }
  44. if !enabled || status == "offline" {
  45. return nil, false, true, nil
  46. }
  47. rt, err := s.runtimeFor(ib)
  48. if err != nil {
  49. return nil, false, true, nil
  50. }
  51. return rt, true, false, nil
  52. }
  53. func (s *InboundService) NodeIsPending(nodeID *int) bool {
  54. if nodeID == nil {
  55. return false
  56. }
  57. return (&NodeService{}).IsNodePending(*nodeID)
  58. }
  59. func (s *InboundService) AnyNodePending(inboundIds []int) bool {
  60. if len(inboundIds) == 0 {
  61. return false
  62. }
  63. nodeSvc := NodeService{}
  64. for _, id := range inboundIds {
  65. ib, err := s.GetInbound(id)
  66. if err != nil || ib.NodeID == nil {
  67. continue
  68. }
  69. if nodeSvc.IsNodePending(*ib.NodeID) {
  70. return true
  71. }
  72. }
  73. return false
  74. }
  75. // ReconcileNode pushes every inbound and sweeps undesired remote tags even when
  76. // individual operations fail, returning the failures joined: one inbound the
  77. // node rejects (e.g. a legacy protocol failing validation, #5685) must not
  78. // stall the rest of the node's config — or, via syncOne, its traffic sync.
  79. func (s *InboundService) ReconcileNode(ctx context.Context, rt *runtime.Remote, n *model.Node) error {
  80. if rt == nil || n == nil || n.Id <= 0 {
  81. return nil
  82. }
  83. nodeID := n.Id
  84. db := database.GetDB()
  85. var inbounds []*model.Inbound
  86. if err := db.Model(model.Inbound{}).Where("node_id = ?", nodeID).Find(&inbounds).Error; err != nil {
  87. return err
  88. }
  89. remoteInbounds, err := rt.ListInboundOptions(ctx)
  90. if err != nil {
  91. return err
  92. }
  93. remoteTags := make([]string, 0, len(remoteInbounds))
  94. remoteTagSet := make(map[string]struct{}, len(remoteTags))
  95. for _, remoteIb := range remoteInbounds {
  96. remoteTags = append(remoteTags, remoteIb.Tag)
  97. remoteTagSet[remoteIb.Tag] = struct{}{}
  98. }
  99. prefix := nodeTagPrefix(&nodeID)
  100. desiredTags := make(map[string]struct{}, len(inbounds)*2)
  101. var errs []error
  102. for _, ib := range inbounds {
  103. desiredTags[ib.Tag] = struct{}{}
  104. // existsOnNode: does the node already report this inbound under any of the
  105. // tag forms it may be stored as? If so, an unchanged push can be skipped.
  106. _, existsOnNode := remoteTagSet[ib.Tag]
  107. if prefix != "" {
  108. if stripped, found := strings.CutPrefix(ib.Tag, prefix); found {
  109. desiredTags[stripped] = struct{}{}
  110. if _, ok := remoteTagSet[stripped]; ok {
  111. existsOnNode = true
  112. }
  113. } else {
  114. desiredTags[prefix+ib.Tag] = struct{}{}
  115. if _, ok := remoteTagSet[prefix+ib.Tag]; ok {
  116. existsOnNode = true
  117. }
  118. }
  119. }
  120. runtimeIb := ib
  121. if built, bErr := s.buildInboundForNodePush(db, ib); bErr == nil {
  122. runtimeIb = built
  123. }
  124. if !existsOnNode && n.Guid != "" && ib.OriginNodeGuid == n.Guid {
  125. var compatible []runtime.RemoteInboundOption
  126. for _, remoteIb := range remoteInbounds {
  127. if remoteIb.Port == runtimeIb.Port &&
  128. remoteIb.Protocol == runtimeIb.Protocol &&
  129. strings.TrimSpace(remoteIb.Listen) == strings.TrimSpace(runtimeIb.Listen) {
  130. compatible = append(compatible, remoteIb)
  131. }
  132. }
  133. switch len(compatible) {
  134. case 1:
  135. alias := compatible[0]
  136. desiredTags[alias.Tag] = struct{}{}
  137. rt.AdoptInboundAlias(runtimeIb, alias)
  138. existsOnNode = true
  139. logger.Infof("adopted compatible inbound %q on node %s as %q", alias.Tag, n.Name, ib.Tag)
  140. case 0:
  141. // No compatible occupant: keep the normal create path, which
  142. // leaves a real port/protocol drift loud.
  143. default:
  144. for _, candidate := range compatible {
  145. desiredTags[candidate.Tag] = struct{}{}
  146. }
  147. errs = append(errs, fmt.Errorf("reconcile inbound %q: ambiguous compatible remote inbounds", ib.Tag))
  148. continue
  149. }
  150. }
  151. if _, err := rt.ReconcileInbound(ctx, runtimeIb, existsOnNode); err != nil {
  152. errs = append(errs, fmt.Errorf("reconcile inbound %q: %w", ib.Tag, err))
  153. }
  154. }
  155. // Before the first clean sync adopts the node's inbounds, "absent locally"
  156. // means "not imported yet" — sweeping now would wipe the node at onboarding.
  157. if n.InboundsAdoptedAt == 0 {
  158. return errors.Join(errs...)
  159. }
  160. // In "selected" sync mode the panel only manages the selected tags: the
  161. // rest were never imported, so their absence from the local DB must not
  162. // delete them from the node. Only a selected tag missing locally (the
  163. // panel deleted it while the node was unreachable) may be swept.
  164. var selected map[string]struct{}
  165. if n.InboundSyncMode == "selected" {
  166. selected = make(map[string]struct{}, len(n.InboundTags))
  167. for _, tag := range n.InboundTags {
  168. selected[tag] = struct{}{}
  169. }
  170. }
  171. for _, tag := range remoteTags {
  172. if _, want := desiredTags[tag]; want {
  173. continue
  174. }
  175. if selected != nil {
  176. if _, managed := selected[tag]; !managed {
  177. continue
  178. }
  179. }
  180. if err := rt.DelInbound(ctx, &model.Inbound{Tag: tag}); err != nil {
  181. errs = append(errs, fmt.Errorf("reconcile delete %q: %w", tag, err))
  182. }
  183. }
  184. return errors.Join(errs...)
  185. }
  186. const resetGracePeriodMs int64 = 30000
  187. // onlineGracePeriodMs must comfortably exceed the 5s traffic-poll interval —
  188. // Xray's stats counters often report a zero delta for an active session across
  189. // a single poll, so a 5s grace would still drop the client on the next tick.
  190. // ~4 polls of slack keeps idle-but-connected clients visible without lingering
  191. // long after a real disconnect.
  192. const onlineGracePeriodMs int64 = 20000
  193. type nodeTrafficCounter struct {
  194. Up int64
  195. Down int64
  196. }
  197. func (s *InboundService) upsertNodeBaseline(tx *gorm.DB, nodeID int, email string, up, down int64) error {
  198. return tx.Clauses(clause.OnConflict{
  199. Columns: []clause.Column{{Name: "node_id"}, {Name: "email"}},
  200. DoUpdates: clause.AssignmentColumns([]string{"up", "down"}),
  201. }).Create(&model.NodeClientTraffic{NodeId: nodeID, Email: email, Up: up, Down: down}).Error
  202. }
  203. // mergeActivationExpiry: master absolute wins; node may only activate when
  204. // master is unset/duration. Node auto-renew goes through nodeClientRenewed.
  205. func mergeActivationExpiry(existing, node int64) int64 {
  206. if existing > 0 {
  207. return existing
  208. }
  209. return node
  210. }
  211. // masterLimitsAllowClient reports whether the master's own deadline and quota
  212. // (including this tick's deltas) still permit the client.
  213. func masterLimitsAllowClient(master *xray.ClientTraffic, now, deltaUp, deltaDown int64) bool {
  214. if master == nil {
  215. return false
  216. }
  217. if master.ExpiryTime > 0 && master.ExpiryTime <= now {
  218. return false
  219. }
  220. if master.Total > 0 && master.Up+deltaUp+master.Down+deltaDown >= master.Total {
  221. return false
  222. }
  223. return true
  224. }
  225. // nodeDisableIsStale reports an enable=false the node decided against limits the
  226. // master has since changed, so it must not latch back (#6228 / #4917).
  227. func nodeDisableIsStale(master *xray.ClientTraffic, node xray.ClientTraffic, now, deltaUp, deltaDown int64) bool {
  228. if master == nil {
  229. return false
  230. }
  231. // Matching limits mean the node judged the client on the master's own terms:
  232. // that verdict is genuine and still latches, as #4917 requires.
  233. if node.ExpiryTime == master.ExpiryTime && node.Total == master.Total {
  234. return false
  235. }
  236. return masterLimitsAllowClient(master, now, deltaUp, deltaDown)
  237. }
  238. func clampTrafficCounter(v int64) int64 {
  239. if v > database.TrafficMax {
  240. return database.TrafficMax
  241. }
  242. if v < 0 {
  243. return 0
  244. }
  245. return v
  246. }
  247. // applyMasterClientLifecycle overlays the already-merged master row onto a
  248. // node-reported client for SyncInbound (#6228).
  249. func applyMasterClientLifecycle(c *model.Client, master *xray.ClientTraffic, cs *xray.ClientTraffic) {
  250. if master == nil {
  251. // No central row to speak for the client: the node's own latch is all
  252. // there is, and it may only disable.
  253. if cs != nil && !cs.Enable {
  254. c.Enable = false
  255. }
  256. return
  257. }
  258. c.ExpiryTime = mergeActivationExpiry(master.ExpiryTime, c.ExpiryTime)
  259. c.Enable = master.Enable
  260. }
  261. // nodeClientRenewed reports a node-side auto-renew: an absolute deadline moved
  262. // forward, evidenced by a renewal-count bump or a drop below the stored baseline.
  263. func nodeClientRenewed(existing *xray.ClientTraffic, cs xray.ClientTraffic, canon, base nodeTrafficCounter) bool {
  264. if (cs.Reset <= 0 && cs.ResetDay <= 0) || cs.ExpiryTime <= 0 || existing.ExpiryTime <= 0 {
  265. return false
  266. }
  267. if cs.ExpiryTime <= existing.ExpiryTime {
  268. return false
  269. }
  270. // A client that used no traffic in the period never dips, so the renewal
  271. // counter is the only evidence autoRenewClients leaves behind (#6228).
  272. if cs.ResetCount > existing.ResetCount {
  273. return true
  274. }
  275. return canon.Up < base.Up || canon.Down < base.Down
  276. }
  277. // liftActivatedClientRecordExpiries copies a node-activated deadline from
  278. // client_traffics onto client records still holding the negative duration (#5714).
  279. func liftActivatedClientRecordExpiries(tx *gorm.DB) error {
  280. return tx.Exec(
  281. `UPDATE clients
  282. SET expiry_time = (SELECT ct.expiry_time FROM client_traffics ct WHERE ct.email = clients.email AND ct.expiry_time > 0 LIMIT 1)
  283. WHERE clients.expiry_time < 0
  284. AND EXISTS (SELECT 1 FROM client_traffics ct WHERE ct.email = clients.email AND ct.expiry_time > 0)`,
  285. ).Error
  286. }
  287. // SnapshotHasUnadoptedInbounds reports whether the snapshot carries a tag with
  288. // no central row yet, i.e. the next merge would adopt a new inbound.
  289. func (s *InboundService) SnapshotHasUnadoptedInbounds(nodeID int, snap *runtime.TrafficSnapshot) (bool, error) {
  290. if snap == nil || len(snap.Inbounds) == 0 {
  291. return false, nil
  292. }
  293. var tags []string
  294. if err := database.GetDB().Model(model.Inbound{}).
  295. Where("node_id = ?", nodeID).
  296. Pluck("tag", &tags).Error; err != nil {
  297. return false, err
  298. }
  299. prefix := nodeTagPrefix(&nodeID)
  300. known := make(map[string]struct{}, len(tags)*2)
  301. for _, tag := range tags {
  302. known[tag] = struct{}{}
  303. if prefix != "" {
  304. if stripped, found := strings.CutPrefix(tag, prefix); found {
  305. known[stripped] = struct{}{}
  306. } else {
  307. known[prefix+tag] = struct{}{}
  308. }
  309. }
  310. }
  311. for _, ib := range snap.Inbounds {
  312. if ib == nil {
  313. continue
  314. }
  315. if _, ok := known[ib.Tag]; !ok {
  316. return true, nil
  317. }
  318. }
  319. return false, nil
  320. }
  321. // SetRemoteTraffic merges a node snapshot. justPushed marks the tick whose
  322. // config push just landed, whose snapshot may still predate it (#6228).
  323. func (s *InboundService) SetRemoteTraffic(nodeID int, snap *runtime.TrafficSnapshot, dirty, justPushed bool) (bool, error) {
  324. var structuralChange bool
  325. err := submitTrafficWrite(func() error {
  326. var inner error
  327. structuralChange, inner = s.setRemoteTrafficLocked(nodeID, snap, dirty, justPushed)
  328. return inner
  329. })
  330. return structuralChange, err
  331. }
  332. // GetNodeInboundTrafficTotals returns the current cumulative up/down for every
  333. // node-hosted inbound, keyed by tag. The node sync diffs successive snapshots of
  334. // this to derive per-inbound speed for the dashboard — node inbounds have no
  335. // local Xray poll to produce live deltas the way local inbounds do.
  336. func (s *InboundService) GetNodeInboundTrafficTotals() (map[string][2]int64, error) {
  337. var rows []struct {
  338. Tag string
  339. Up int64
  340. Down int64
  341. }
  342. if err := database.GetDB().Table("inbounds").
  343. Select("tag, up, down").
  344. Where("node_id IS NOT NULL").
  345. Scan(&rows).Error; err != nil {
  346. return nil, err
  347. }
  348. out := make(map[string][2]int64, len(rows))
  349. for _, r := range rows {
  350. out[r.Tag] = [2]int64{r.Up, r.Down}
  351. }
  352. return out, nil
  353. }
  354. func adoptedWireChanged(c, snapIb *model.Inbound, adoptedSettings string) bool {
  355. return c.Settings != adoptedSettings ||
  356. c.Enable != snapIb.Enable ||
  357. c.Remark != snapIb.Remark ||
  358. c.SubSortIndex != normalizeSubSortIndex(snapIb.SubSortIndex) ||
  359. c.Listen != snapIb.Listen ||
  360. c.Port != snapIb.Port ||
  361. c.Protocol != snapIb.Protocol ||
  362. c.Total != snapIb.Total ||
  363. c.ExpiryTime != snapIb.ExpiryTime ||
  364. c.StreamSettings != snapIb.StreamSettings ||
  365. c.Sniffing != snapIb.Sniffing ||
  366. c.TrafficReset != snapIb.TrafficReset ||
  367. c.TrafficResetDay != normalizeTrafficResetDay(snapIb.TrafficResetDay)
  368. }
  369. // adoptedWireInbound is the central inbound as it reads after adopting the
  370. // node-reported wire fields — the payload the reconcile fingerprint must track.
  371. func adoptedWireInbound(c, snapIb *model.Inbound, adoptedSettings string) *model.Inbound {
  372. a := *c
  373. a.Enable = snapIb.Enable
  374. a.Remark = snapIb.Remark
  375. a.SubSortIndex = normalizeSubSortIndex(snapIb.SubSortIndex)
  376. a.Listen = snapIb.Listen
  377. a.Port = snapIb.Port
  378. a.Protocol = snapIb.Protocol
  379. a.Total = snapIb.Total
  380. a.ExpiryTime = snapIb.ExpiryTime
  381. a.Settings = adoptedSettings
  382. a.StreamSettings = snapIb.StreamSettings
  383. a.Sniffing = snapIb.Sniffing
  384. a.TrafficReset = snapIb.TrafficReset
  385. a.TrafficResetDay = normalizeTrafficResetDay(snapIb.TrafficResetDay)
  386. return &a
  387. }
  388. func (s *InboundService) setRemoteTrafficLocked(nodeID int, snap *runtime.TrafficSnapshot, dirty, justPushed bool) (bool, error) {
  389. if snap == nil || nodeID <= 0 {
  390. return false, nil
  391. }
  392. db := database.GetDB()
  393. now := time.Now().UnixMilli()
  394. // originGuidFor attributes a synced inbound to the panel that physically
  395. // hosts it. A node's OWN inbounds report either an empty origin or — on
  396. // builds that set it locally — the node's own panelGuid; both resolve to
  397. // selfKey, which is the node's panelGuid unless that GUID is ambiguous
  398. // (shared with another node or the master, i.e. a cloned server), in which
  399. // case it falls back to the node-unique id so #4983 attribution doesn't
  400. // collapse two physical nodes into one bucket. Only a DIFFERENT, non-empty
  401. // origin (an inbound the node forwards from its own sub-node) is kept as-is,
  402. // so a chained Node1->Node2->Node3 still attributes Node3's inbounds to Node3.
  403. var nodeRow model.Node
  404. db.Select("guid", "config_dirty", "inbound_sync_mode", "inbound_tags").Where("id = ?", nodeID).First(&nodeRow)
  405. // Re-read inside the serialized writer: a client added while this snapshot
  406. // was in flight marks the node dirty after the caller sampled the flag.
  407. dirty = dirty || nodeRow.ConfigDirty
  408. // Adoption, record sync and sweeps still run on a just-pushed tick; only the
  409. // client lifecycle merge waits for a snapshot that reflects the push.
  410. lifecycleFrozen := dirty || justPushed
  411. nodeRow.Id = nodeID
  412. unmanagedTag := unmanagedTagPredicate(&nodeRow)
  413. selfKey := effectiveNodeKey(&model.Node{Id: nodeID, Guid: nodeRow.Guid})
  414. guidShared := nodeRow.Guid != "" && selfKey != nodeRow.Guid
  415. originGuidFor := func(snapIb *model.Inbound) string {
  416. if snapIb.OriginNodeGuid != "" && snapIb.OriginNodeGuid != nodeRow.Guid {
  417. return snapIb.OriginNodeGuid
  418. }
  419. return selfKey
  420. }
  421. var central []model.Inbound
  422. if err := db.Model(model.Inbound{}).
  423. Where("node_id = ?", nodeID).
  424. Find(&central).Error; err != nil {
  425. return false, err
  426. }
  427. // Index under the stored tag and its prefix-flipped form so a snap matches
  428. // whether the n<id>- prefix lives on the node side, the central side, or
  429. // neither — a mismatch must never spawn a duplicate central inbound.
  430. tagToCentral := make(map[string]*model.Inbound, len(central)*2)
  431. prefix := nodeTagPrefix(&nodeID)
  432. for i := range central {
  433. tagToCentral[central[i].Tag] = &central[i]
  434. if prefix != "" {
  435. if stripped, found := strings.CutPrefix(central[i].Tag, prefix); found {
  436. tagToCentral[stripped] = &central[i]
  437. } else {
  438. tagToCentral[prefix+central[i].Tag] = &central[i]
  439. }
  440. }
  441. }
  442. var centralClientStats []xray.ClientTraffic
  443. if len(central) > 0 {
  444. ids := make([]int, 0, len(central))
  445. for i := range central {
  446. ids = append(ids, central[i].Id)
  447. }
  448. if err := db.Model(xray.ClientTraffic{}).
  449. Where("inbound_id IN ?", ids).
  450. Find(&centralClientStats).Error; err != nil {
  451. return false, err
  452. }
  453. }
  454. type csKey struct {
  455. inboundID int
  456. email string
  457. }
  458. centralCS := make(map[csKey]*xray.ClientTraffic, len(centralClientStats))
  459. centralCSByEmail := make(map[string]*xray.ClientTraffic, len(centralClientStats))
  460. for i := range centralClientStats {
  461. centralCS[csKey{centralClientStats[i].InboundId, centralClientStats[i].Email}] = &centralClientStats[i]
  462. centralCSByEmail[centralClientStats[i].Email] = &centralClientStats[i]
  463. }
  464. nodeBaselines := make(map[string]nodeTrafficCounter)
  465. var baselineRows []model.NodeClientTraffic
  466. if err := db.Model(&model.NodeClientTraffic{}).
  467. Where("node_id = ?", nodeID).
  468. Find(&baselineRows).Error; err != nil {
  469. return false, err
  470. }
  471. for i := range baselineRows {
  472. nodeBaselines[baselineRows[i].Email] = nodeTrafficCounter{Up: baselineRows[i].Up, Down: baselineRows[i].Down}
  473. }
  474. var defaultUserId int
  475. if len(central) > 0 {
  476. defaultUserId = central[0].UserId
  477. } else {
  478. var u model.User
  479. if err := db.Model(model.User{}).Order("id asc").First(&u).Error; err == nil {
  480. defaultUserId = u.Id
  481. } else {
  482. defaultUserId = 1
  483. }
  484. }
  485. // Union of every email the snapshot still reports, across all inbounds.
  486. // The (node, email) baseline rows are keyed per node, not per inbound, so
  487. // the sweeps below must only drop one when the email left the node
  488. // entirely — an email whose stats moved to (or always lived under) a
  489. // sibling inbound still needs its baseline for the sibling's delta
  490. // computation (#5202).
  491. //
  492. // Xray counts traffic per email, not per inbound, so a multi-attached
  493. // client's shared counter is copied onto every inbound it's on. Fold each
  494. // email to its per-field max (nodeEmailTotals) so divergent copies can't make
  495. // the reset clamp re-add a lower sibling as fresh traffic (#5274).
  496. snapEmailsAll := make(map[string]struct{})
  497. nodeEmailTotals := make(map[string]nodeTrafficCounter)
  498. for _, snapIb := range snap.Inbounds {
  499. if snapIb == nil {
  500. continue
  501. }
  502. for i := range snapIb.ClientStats {
  503. email := snapIb.ClientStats[i].Email
  504. snapEmailsAll[email] = struct{}{}
  505. cur := nodeEmailTotals[email]
  506. if snapIb.ClientStats[i].Up > cur.Up {
  507. cur.Up = snapIb.ClientStats[i].Up
  508. }
  509. if snapIb.ClientStats[i].Down > cur.Down {
  510. cur.Down = snapIb.ClientStats[i].Down
  511. }
  512. nodeEmailTotals[email] = cur
  513. }
  514. }
  515. // Membership set for the rowExists checks below. Only the snapshot's emails
  516. // are ever probed, so scope the lookup to those instead of plucking the whole
  517. // client_traffics table (50k+ rows) on every node poll.
  518. existingEmails := make(map[string]struct{}, len(snapEmailsAll))
  519. if len(snapEmailsAll) > 0 {
  520. snapEmailList := make([]string, 0, len(snapEmailsAll))
  521. for email := range snapEmailsAll {
  522. snapEmailList = append(snapEmailList, email)
  523. }
  524. for _, batch := range chunkStrings(snapEmailList, sqliteMaxVars) {
  525. var found []string
  526. if err := db.Model(xray.ClientTraffic{}).Where("email IN ?", batch).Pluck("email", &found).Error; err != nil {
  527. return false, err
  528. }
  529. for _, e := range found {
  530. existingEmails[e] = struct{}{}
  531. }
  532. }
  533. }
  534. tx := db.Begin()
  535. committed := false
  536. defer func() {
  537. if !committed {
  538. tx.Rollback()
  539. }
  540. }()
  541. structuralChange := false
  542. lifecycleLifted := false
  543. var adoptedInbounds []*model.Inbound
  544. type pendingAdopt struct {
  545. central *model.Inbound
  546. snapIb *model.Inbound
  547. wireSettings string
  548. }
  549. var pendingAdopts []pendingAdopt
  550. newInboundIDs := make(map[int]struct{})
  551. snapTags := make(map[string]struct{}, len(snap.Inbounds))
  552. for _, snapIb := range snap.Inbounds {
  553. if snapIb == nil {
  554. continue
  555. }
  556. snapTags[snapIb.Tag] = struct{}{}
  557. // Record the prefix-flipped form too so the orphan sweep below keeps a
  558. // central inbound whether its tag carries the n<id>- prefix or not.
  559. if prefix != "" {
  560. if stripped, found := strings.CutPrefix(snapIb.Tag, prefix); found {
  561. snapTags[stripped] = struct{}{}
  562. } else {
  563. snapTags[prefix+snapIb.Tag] = struct{}{}
  564. }
  565. }
  566. c, ok := tagToCentral[snapIb.Tag]
  567. if !ok {
  568. origin := originGuidFor(snapIb)
  569. var compatible []*model.Inbound
  570. for i := range central {
  571. candidate := &central[i]
  572. if candidate.OriginNodeGuid == origin &&
  573. candidate.Port == snapIb.Port &&
  574. candidate.Protocol == snapIb.Protocol &&
  575. strings.TrimSpace(candidate.Listen) == strings.TrimSpace(snapIb.Listen) {
  576. compatible = append(compatible, candidate)
  577. }
  578. }
  579. switch len(compatible) {
  580. case 1:
  581. c, ok = compatible[0], true
  582. tagToCentral[snapIb.Tag] = c
  583. snapTags[c.Tag] = struct{}{}
  584. case 0:
  585. // A genuinely new inbound follows the normal adoption path.
  586. default:
  587. return false, fmt.Errorf("setRemoteTraffic: inbound %q has ambiguous compatible central aliases", snapIb.Tag)
  588. }
  589. }
  590. if !ok {
  591. if dirty {
  592. continue
  593. }
  594. // Try snap.Tag first; on collision fall back to the n<id>-
  595. // prefixed form so local+node can both own the same port.
  596. pickFreeTag := func() (string, error) {
  597. candidates := []string{snapIb.Tag}
  598. if prefix != "" && !strings.HasPrefix(snapIb.Tag, prefix) {
  599. candidates = append(candidates, prefix+snapIb.Tag)
  600. }
  601. for _, t := range candidates {
  602. var owner model.Inbound
  603. err := tx.Where("tag = ?", t).First(&owner).Error
  604. if errors.Is(err, gorm.ErrRecordNotFound) {
  605. return t, nil
  606. }
  607. if err != nil {
  608. return "", err
  609. }
  610. }
  611. return "", nil
  612. }
  613. chosenTag, err := pickFreeTag()
  614. if err != nil {
  615. logger.Warningf("setRemoteTraffic: check tag %q failed: %v", snapIb.Tag, err)
  616. continue
  617. }
  618. if chosenTag == "" {
  619. key := fmt.Sprintf("%d:%s", nodeID, snapIb.Tag)
  620. if _, seen := reportedRemoteTagConflict.LoadOrStore(key, struct{}{}); !seen {
  621. logger.Warningf(
  622. "setRemoteTraffic: tag %q from node %d collides with an existing inbound even after the n%d- prefix — skipping (rename one side to remove the duplicate)",
  623. snapIb.Tag, nodeID, nodeID,
  624. )
  625. }
  626. continue
  627. }
  628. reportedRemoteTagConflict.Delete(fmt.Sprintf("%d:%s", nodeID, snapIb.Tag))
  629. newIb := model.Inbound{
  630. UserId: defaultUserId,
  631. NodeID: &nodeID,
  632. OriginNodeGuid: originGuidFor(snapIb),
  633. Tag: chosenTag,
  634. Listen: snapIb.Listen,
  635. Port: snapIb.Port,
  636. Protocol: snapIb.Protocol,
  637. Settings: snapIb.Settings,
  638. StreamSettings: snapIb.StreamSettings,
  639. Sniffing: snapIb.Sniffing,
  640. TrafficReset: snapIb.TrafficReset,
  641. TrafficResetDay: normalizeTrafficResetDay(snapIb.TrafficResetDay),
  642. LastTrafficResetTime: snapIb.LastTrafficResetTime,
  643. Enable: snapIb.Enable,
  644. Remark: snapIb.Remark,
  645. SubSortIndex: normalizeSubSortIndex(snapIb.SubSortIndex),
  646. Total: snapIb.Total,
  647. ExpiryTime: snapIb.ExpiryTime,
  648. Up: snapIb.Up,
  649. Down: snapIb.Down,
  650. ShareAddrStrategy: "node",
  651. DisableFlow: snapIb.DisableFlow,
  652. }
  653. if err := tx.Create(&newIb).Error; err != nil {
  654. logger.Warningf("setRemoteTraffic: create central inbound for tag %q failed: %v", snapIb.Tag, err)
  655. continue
  656. }
  657. tagToCentral[snapIb.Tag] = &newIb
  658. if newIb.Tag != snapIb.Tag {
  659. tagToCentral[newIb.Tag] = &newIb
  660. }
  661. if rows := adoptedHostRows(snap.HostGroups, snapIb.Id, newIb.Id); len(rows) > 0 {
  662. if err := tx.Create(&rows).Error; err != nil {
  663. logger.Warningf("setRemoteTraffic: adopt host rows for tag %q failed: %v", newIb.Tag, err)
  664. }
  665. }
  666. newInboundIDs[newIb.Id] = struct{}{}
  667. structuralChange = true
  668. continue
  669. }
  670. inGrace := c.LastTrafficResetTime > 0 && now-c.LastTrafficResetTime < resetGracePeriodMs
  671. // Adopting the node's settings verbatim would re-add a client the master
  672. // deleted moments ago if this snapshot was fetched before the deletion
  673. // push landed — filter just-deleted emails out while their tombstone lives.
  674. adoptedSettings := snapIb.Settings
  675. if stripped, changed := stripTombstonedClients(adoptedSettings); changed {
  676. adoptedSettings = stripped
  677. }
  678. if deduped, changed := dedupeSettingsClients(adoptedSettings); changed {
  679. adoptedSettings = deduped
  680. }
  681. updates := map[string]any{}
  682. if !dirty {
  683. // Defer lifecycle lift until after client_traffics absorbs this tick's
  684. // deltas so quota stale-disable matches SQL (#6228).
  685. pendingAdopts = append(pendingAdopts, pendingAdopt{
  686. central: c, snapIb: snapIb, wireSettings: adoptedSettings,
  687. })
  688. updates["enable"] = snapIb.Enable
  689. updates["remark"] = snapIb.Remark
  690. updates["sub_sort_index"] = normalizeSubSortIndex(snapIb.SubSortIndex)
  691. updates["listen"] = snapIb.Listen
  692. updates["port"] = snapIb.Port
  693. updates["protocol"] = snapIb.Protocol
  694. updates["total"] = snapIb.Total
  695. updates["expiry_time"] = snapIb.ExpiryTime
  696. updates["stream_settings"] = snapIb.StreamSettings
  697. updates["sniffing"] = snapIb.Sniffing
  698. updates["traffic_reset"] = snapIb.TrafficReset
  699. updates["traffic_reset_day"] = normalizeTrafficResetDay(snapIb.TrafficResetDay)
  700. updates["last_traffic_reset_time"] = snapIb.LastTrafficResetTime
  701. }
  702. if !inGrace || (snapIb.Up+snapIb.Down) <= (c.Up+c.Down) {
  703. updates["up"] = snapIb.Up
  704. updates["down"] = snapIb.Down
  705. }
  706. // Physical-home attribution is independent of config-dirty state, so
  707. // keep it current even while the node has pending offline edits. Writes
  708. // once to backfill an existing row, then stays equal (#4983).
  709. if og := originGuidFor(snapIb); c.OriginNodeGuid != og {
  710. updates["origin_node_guid"] = og
  711. }
  712. if !dirty && (c.Remark != snapIb.Remark ||
  713. c.Listen != snapIb.Listen ||
  714. c.Port != snapIb.Port ||
  715. c.Total != snapIb.Total ||
  716. c.ExpiryTime != snapIb.ExpiryTime ||
  717. c.Enable != snapIb.Enable) {
  718. structuralChange = true
  719. }
  720. if len(updates) > 0 {
  721. if err := tx.Model(model.Inbound{}).
  722. Where("id = ?", c.Id).
  723. Updates(updates).Error; err != nil {
  724. return false, err
  725. }
  726. }
  727. }
  728. for _, c := range central {
  729. if dirty {
  730. continue
  731. }
  732. // A node inbound created disabled is never delivered, so its absence from
  733. // the snapshot is ambiguous rather than evidence of a node-side delete.
  734. if !c.Enable {
  735. continue
  736. }
  737. if len(snapTags) == 0 {
  738. // A node mid-restart or with a transient DB error can return an empty
  739. // inbound list with success=true. Treat "zero inbounds reported" as
  740. // "nothing to say", not "delete all my inbounds" — otherwise a blip
  741. // wipes the node's central inbounds and every client on them (and
  742. // resets traffic history on re-create). A real per-inbound deletion
  743. // still sweeps, because the node keeps reporting its other inbounds.
  744. continue
  745. }
  746. if _, kept := snapTags[c.Tag]; kept {
  747. continue
  748. }
  749. if unmanagedTag(c.Tag) {
  750. continue
  751. }
  752. // This drops the central inbound and its clients' traffic history, so say
  753. // so: silent removal is indistinguishable from an inbound never arriving.
  754. logger.Warningf("setRemoteTraffic: node %d no longer reports inbound %q (id %d, port %d) — removing it centrally", nodeID, c.Tag, c.Id, c.Port)
  755. var goneEmails []string
  756. if err := tx.Model(xray.ClientTraffic{}).
  757. Where("inbound_id = ?", c.Id).
  758. Pluck("email", &goneEmails).Error; err != nil {
  759. return false, err
  760. }
  761. if len(goneEmails) > 0 {
  762. // Baselines are per (node, email), not per inbound: keep them for
  763. // emails the snapshot still reports under a sibling inbound (#5202).
  764. baselineGone := make([]string, 0, len(goneEmails))
  765. for _, e := range goneEmails {
  766. if _, still := snapEmailsAll[e]; !still {
  767. baselineGone = append(baselineGone, e)
  768. }
  769. }
  770. // Chunk to avoid SQLite bind var limit when a node has many clients
  771. // removed (e.g. after API bulk delete or structural change on node inbound).
  772. for _, batch := range chunkStrings(baselineGone, sqliteMaxVars) {
  773. if err := tx.Where("node_id = ? AND email IN ?", nodeID, batch).
  774. Delete(&model.NodeClientTraffic{}).Error; err != nil {
  775. return false, err
  776. }
  777. }
  778. // The per-email row is the shared accumulator across every inbound
  779. // (and node) the email is attached to. Only drop it when this was the
  780. // email's last inbound — wiping it while a sibling still feeds it
  781. // loses the summed history, and the next node sync would re-seed the
  782. // row with that node's counter alone.
  783. sharedEmails, sErr := s.emailsUsedByOtherInbounds(goneEmails, c.Id)
  784. if sErr != nil {
  785. return false, sErr
  786. }
  787. delEmails := make([]string, 0, len(goneEmails))
  788. for _, e := range goneEmails {
  789. if !sharedEmails[strings.ToLower(strings.TrimSpace(e))] {
  790. delEmails = append(delEmails, e)
  791. }
  792. }
  793. for _, batch := range chunkStrings(delEmails, sqliteMaxVars) {
  794. if err := tx.Where("inbound_id = ? AND email IN ?", c.Id, batch).
  795. Delete(&xray.ClientTraffic{}).Error; err != nil {
  796. return false, err
  797. }
  798. }
  799. }
  800. if err := s.clientService.DetachInbound(tx, c.Id); err != nil {
  801. return false, err
  802. }
  803. if err := tx.Where("id = ?", c.Id).
  804. Delete(&model.Inbound{}).Error; err != nil {
  805. return false, err
  806. }
  807. delete(tagToCentral, c.Tag)
  808. structuralChange = true
  809. }
  810. for _, snapIb := range snap.Inbounds {
  811. if snapIb == nil {
  812. continue
  813. }
  814. c, ok := tagToCentral[snapIb.Tag]
  815. if !ok {
  816. continue
  817. }
  818. snapEmails := make(map[string]struct{}, len(snapIb.ClientStats))
  819. // Parsed once per inbound on the first renewal candidate, not per client.
  820. var snapExpiries map[string]int64
  821. for _, cs := range snapIb.ClientStats {
  822. snapEmails[cs.Email] = struct{}{}
  823. // Node-wide total, not this inbound's possibly-stale copy (#5274).
  824. canon := nodeEmailTotals[cs.Email]
  825. base, seen := nodeBaselines[cs.Email]
  826. var deltaUp, deltaDown int64
  827. if seen {
  828. if deltaUp = canon.Up - base.Up; deltaUp < 0 {
  829. deltaUp = 0
  830. }
  831. if deltaDown = canon.Down - base.Down; deltaDown < 0 {
  832. deltaDown = 0
  833. }
  834. }
  835. if _, rowExists := existingEmails[cs.Email]; !rowExists {
  836. if dirty {
  837. continue
  838. }
  839. _, isNewInbound := newInboundIDs[c.Id]
  840. // On a known inbound a missing row plus a live tombstone means the
  841. // master just deleted this client and the snapshot predates the
  842. // deletion push — recreating the row (at zero) would resurrect the
  843. // client. A freshly adopted inbound still gets its row (seeded at
  844. // zero) so adoption semantics stay intact.
  845. if !isNewInbound && isClientEmailTombstoned(cs.Email) {
  846. continue
  847. }
  848. var seedUp, seedDown int64
  849. if isNewInbound && !isClientEmailTombstoned(cs.Email) {
  850. seedUp, seedDown = canon.Up, canon.Down
  851. }
  852. row := &xray.ClientTraffic{
  853. InboundId: c.Id,
  854. Email: cs.Email,
  855. Enable: cs.Enable,
  856. Total: cs.Total,
  857. ExpiryTime: cs.ExpiryTime,
  858. Reset: cs.Reset,
  859. ResetDay: cs.ResetDay,
  860. Up: seedUp,
  861. Down: seedDown,
  862. LastOnline: cs.LastOnline,
  863. }
  864. if err := tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "email"}}, DoNothing: true}).
  865. Create(row).Error; err != nil {
  866. return false, err
  867. }
  868. centralCS[csKey{c.Id, cs.Email}] = row
  869. centralCSByEmail[cs.Email] = row
  870. existingEmails[cs.Email] = struct{}{}
  871. structuralChange = true
  872. if err := s.upsertNodeBaseline(tx, nodeID, cs.Email, canon.Up, canon.Down); err != nil {
  873. return false, err
  874. }
  875. nodeBaselines[cs.Email] = nodeTrafficCounter{Up: canon.Up, Down: canon.Down}
  876. continue
  877. }
  878. existing := centralCSByEmail[cs.Email]
  879. if existing != nil {
  880. expiryChanged := !lifecycleFrozen && existing.ExpiryTime != mergeActivationExpiry(existing.ExpiryTime, cs.ExpiryTime)
  881. // Only a real latch to disabled is structural; one-way merge never
  882. // re-enables from the node.
  883. enableChanged := !lifecycleFrozen && existing.Enable && !cs.Enable &&
  884. !nodeDisableIsStale(existing, cs, now, deltaUp, deltaDown)
  885. metaChanged := !lifecycleFrozen && (existing.Total != cs.Total || existing.Reset != cs.Reset)
  886. if enableChanged || metaChanged || expiryChanged {
  887. structuralChange = true
  888. }
  889. }
  890. renewed := !lifecycleFrozen && seen && existing != nil && nodeClientRenewed(existing, cs, canon, base)
  891. if renewed {
  892. // Reject when the node's own settings still carry the old absolute:
  893. // lagging ClientStats after a master shorten mimic a renew (#6228).
  894. if snapExpiries == nil {
  895. snapExpiries = settingsClientAbsoluteExpiries(snapIb.Settings)
  896. }
  897. if se, ok := snapExpiries[cs.Email]; ok && se <= existing.ExpiryTime {
  898. renewed = false
  899. }
  900. }
  901. if renewed {
  902. // A renewal starts a fresh quota window: adopt the node's counters
  903. // and enable state, drop stale pushes (mirrors autoRenewClients).
  904. if err := tx.Exec(
  905. fmt.Sprintf(
  906. `UPDATE client_traffics
  907. SET up = ?, down = ?, enable = ?, total = ?,
  908. expiry_time = ?, reset = ?, reset_day = ?, reset_count = ?, last_online = %s
  909. WHERE email = ?`,
  910. database.GreatestExpr("last_online", "?"),
  911. ),
  912. canon.Up, canon.Down, cs.Enable, cs.Total,
  913. cs.ExpiryTime, cs.Reset, cs.ResetDay, cs.ResetCount,
  914. cs.LastOnline, cs.Email,
  915. ).Error; err != nil {
  916. return false, err
  917. }
  918. if err := clearGlobalTraffic(tx, cs.Email); err != nil {
  919. return false, err
  920. }
  921. existing.Up = canon.Up
  922. existing.Down = canon.Down
  923. existing.Enable = cs.Enable
  924. existing.Total = cs.Total
  925. existing.ExpiryTime = cs.ExpiryTime
  926. existing.Reset = cs.Reset
  927. existing.ResetCount = cs.ResetCount
  928. structuralChange = true
  929. } else if lifecycleFrozen {
  930. // Push pending or just landed: only counters may move, the master
  931. // keeps expiry/enable/total/reset.
  932. if err := tx.Exec(
  933. fmt.Sprintf(
  934. `UPDATE client_traffics
  935. SET up = %s, down = %s, last_online = %s
  936. WHERE email = ?`,
  937. database.ClampedAddExpr("up"),
  938. database.ClampedAddExpr("down"),
  939. database.GreatestExpr("last_online", "?"),
  940. ),
  941. deltaUp, deltaDown, cs.LastOnline, cs.Email,
  942. ).Error; err != nil {
  943. return false, err
  944. }
  945. if existing != nil {
  946. existing.Up = clampTrafficCounter(existing.Up + deltaUp)
  947. existing.Down = clampTrafficCounter(existing.Down + deltaDown)
  948. }
  949. } else {
  950. enableExpr := database.ClientTrafficEnableMergeExpr()
  951. expiryExpr := database.ClientTrafficExpiryMergeExpr()
  952. if err := tx.Exec(
  953. fmt.Sprintf(
  954. `UPDATE client_traffics
  955. SET up = %s, down = %s, enable = %s, total = ?,
  956. expiry_time = %s,
  957. reset = ?, reset_day = ?, last_online = %s
  958. WHERE email = ?`,
  959. database.ClampedAddExpr("up"),
  960. database.ClampedAddExpr("down"),
  961. enableExpr,
  962. expiryExpr,
  963. database.GreatestExpr("last_online", "?"),
  964. ),
  965. deltaUp, deltaDown,
  966. cs.Enable, cs.ExpiryTime, cs.Total, now, deltaUp, deltaDown,
  967. cs.Total,
  968. cs.ExpiryTime, cs.Reset, cs.ResetDay,
  969. cs.LastOnline, cs.Email,
  970. ).Error; err != nil {
  971. return false, err
  972. }
  973. if existing != nil {
  974. priorExpiry := existing.ExpiryTime
  975. if !cs.Enable && !nodeDisableIsStale(existing, cs, now, deltaUp, deltaDown) {
  976. existing.Enable = false
  977. }
  978. existing.ExpiryTime = mergeActivationExpiry(priorExpiry, cs.ExpiryTime)
  979. existing.Up = clampTrafficCounter(existing.Up + deltaUp)
  980. existing.Down = clampTrafficCounter(existing.Down + deltaDown)
  981. existing.Total = cs.Total
  982. existing.Reset = cs.Reset
  983. }
  984. }
  985. // A dip plus a lagging longer expiry mimics nodeClientRenewed and would
  986. // undo a master shorten once the freeze lifts (#6228).
  987. if lifecycleFrozen && seen && (canon.Up < base.Up || canon.Down < base.Down) {
  988. continue
  989. }
  990. if err := s.upsertNodeBaseline(tx, nodeID, cs.Email, canon.Up, canon.Down); err != nil {
  991. return false, err
  992. }
  993. nodeBaselines[cs.Email] = nodeTrafficCounter{Up: canon.Up, Down: canon.Down}
  994. }
  995. for k, existing := range centralCS {
  996. if dirty {
  997. continue
  998. }
  999. if k.inboundID != c.Id {
  1000. continue
  1001. }
  1002. if _, kept := snapEmails[k.email]; kept {
  1003. continue
  1004. }
  1005. // Gone from this inbound's stats but still reported by the node under
  1006. // a sibling inbound: both the shared accumulator row and the (node,
  1007. // email) baseline must survive, or the sibling's next delta would
  1008. // compute against nothing and freeze the counter (#5202).
  1009. if _, still := snapEmailsAll[k.email]; still {
  1010. continue
  1011. }
  1012. if err := tx.Where("node_id = ? AND email = ?", nodeID, existing.Email).
  1013. Delete(&model.NodeClientTraffic{}).Error; err != nil {
  1014. return false, err
  1015. }
  1016. // Same shared-accumulator rule as the inbound-removal sweep above:
  1017. // keep the row while another inbound still references the email.
  1018. stillUsed, uErr := s.emailUsedByOtherInbounds(existing.Email, c.Id)
  1019. if uErr != nil {
  1020. return false, uErr
  1021. }
  1022. // Usage, quota and expiry live on this row, so a client the orphan
  1023. // sweep will mark keeps it until the reaper confirms the removal.
  1024. if !stillUsed && !clientRecordExists(tx, existing.Email) {
  1025. if err := tx.Where("inbound_id = ? AND email = ?", c.Id, existing.Email).
  1026. Delete(&xray.ClientTraffic{}).Error; err != nil {
  1027. return false, err
  1028. }
  1029. }
  1030. structuralChange = true
  1031. }
  1032. }
  1033. type oldSet struct {
  1034. inboundID int
  1035. emails map[string]struct{}
  1036. }
  1037. var perInboundOld []oldSet
  1038. syncFailedInbounds := map[int]struct{}{}
  1039. for _, p := range pendingAdopts {
  1040. lifted, liftChanged := liftClientLifecycleInSettings(p.wireSettings, centralCSByEmail)
  1041. adoptedSettings := p.wireSettings
  1042. if liftChanged {
  1043. adoptedSettings = lifted
  1044. lifecycleLifted = true
  1045. }
  1046. if p.central.Settings != adoptedSettings {
  1047. if err := tx.Model(model.Inbound{}).
  1048. Where("id = ?", p.central.Id).
  1049. Update("settings", adoptedSettings).Error; err != nil {
  1050. return false, err
  1051. }
  1052. structuralChange = true
  1053. }
  1054. // The fingerprint stamps the un-lifted wire blob on purpose: a lift must
  1055. // leave reconcile a mismatch to re-push against.
  1056. if liftChanged || adoptedWireChanged(p.central, p.snapIb, p.wireSettings) {
  1057. adoptedInbounds = append(adoptedInbounds, adoptedWireInbound(p.central, p.snapIb, p.wireSettings))
  1058. }
  1059. }
  1060. for _, snapIb := range snap.Inbounds {
  1061. if snapIb == nil {
  1062. continue
  1063. }
  1064. c, ok := tagToCentral[snapIb.Tag]
  1065. if !ok {
  1066. continue
  1067. }
  1068. if dirty {
  1069. continue
  1070. }
  1071. var oldEmailsRows []string
  1072. if err := tx.Table("clients").
  1073. Joins("JOIN client_inbounds ON client_inbounds.client_id = clients.id").
  1074. Where("client_inbounds.inbound_id = ?", c.Id).
  1075. Pluck("email", &oldEmailsRows).Error; err == nil {
  1076. oldEmails := make(map[string]struct{}, len(oldEmailsRows))
  1077. for _, e := range oldEmailsRows {
  1078. if e != "" {
  1079. oldEmails[e] = struct{}{}
  1080. }
  1081. }
  1082. perInboundOld = append(perInboundOld, oldSet{inboundID: c.Id, emails: oldEmails})
  1083. }
  1084. clients, gcErr := s.GetClients(snapIb)
  1085. if gcErr != nil {
  1086. logger.Warningf("setRemoteTraffic: parse clients for tag %q failed: %v", snapIb.Tag, gcErr)
  1087. continue
  1088. }
  1089. csByEmail := make(map[string]xray.ClientTraffic, len(snapIb.ClientStats))
  1090. for _, cs := range snapIb.ClientStats {
  1091. csByEmail[cs.Email] = cs
  1092. }
  1093. filtered := clients[:0]
  1094. for i := range clients {
  1095. if isClientEmailTombstoned(clients[i].Email) {
  1096. continue
  1097. }
  1098. existing := centralCSByEmail[clients[i].Email]
  1099. var csPtr *xray.ClientTraffic
  1100. if cs, hit := csByEmail[clients[i].Email]; hit {
  1101. csCopy := cs
  1102. csPtr = &csCopy
  1103. }
  1104. applyMasterClientLifecycle(&clients[i], existing, csPtr)
  1105. filtered = append(filtered, clients[i])
  1106. }
  1107. localEmails := make([]string, 0, len(filtered))
  1108. for i := range filtered {
  1109. if filtered[i].Email != "" {
  1110. localEmails = append(localEmails, filtered[i].Email)
  1111. }
  1112. }
  1113. if len(localEmails) > 0 {
  1114. var localMeta []struct {
  1115. Email string
  1116. Comment string `gorm:"column:comment"`
  1117. }
  1118. if err := tx.Table("clients").
  1119. Select("email, comment").
  1120. Where("email IN ?", localEmails).
  1121. Find(&localMeta).Error; err == nil {
  1122. commentByEmail := make(map[string]string, len(localMeta))
  1123. for _, m := range localMeta {
  1124. commentByEmail[m.Email] = m.Comment
  1125. }
  1126. for i := range filtered {
  1127. if cmt, ok := commentByEmail[filtered[i].Email]; ok {
  1128. filtered[i].Comment = cmt
  1129. }
  1130. }
  1131. }
  1132. }
  1133. if err := s.clientService.SyncInbound(tx, c.Id, filtered); err != nil {
  1134. logger.Warningf("setRemoteTraffic: sync clients for tag %q failed: %v", snapIb.Tag, err)
  1135. syncFailedInbounds[c.Id] = struct{}{}
  1136. }
  1137. }
  1138. for _, old := range perInboundOld {
  1139. // The sweep's premise is that links were just rebuilt from the snapshot,
  1140. // which is exactly what a failed SyncInbound violates.
  1141. if _, failed := syncFailedInbounds[old.inboundID]; failed {
  1142. continue
  1143. }
  1144. var stillAttached []string
  1145. if err := tx.Table("clients").
  1146. Joins("JOIN client_inbounds ON client_inbounds.client_id = clients.id").
  1147. Where("client_inbounds.inbound_id = ?", old.inboundID).
  1148. Pluck("email", &stillAttached).Error; err != nil {
  1149. continue
  1150. }
  1151. stillSet := make(map[string]struct{}, len(stillAttached))
  1152. for _, e := range stillAttached {
  1153. stillSet[e] = struct{}{}
  1154. }
  1155. for email := range old.emails {
  1156. if _, kept := stillSet[email]; kept {
  1157. continue
  1158. }
  1159. var attachmentCount int64
  1160. if err := tx.Table("client_inbounds").
  1161. Joins("JOIN clients ON clients.id = client_inbounds.client_id").
  1162. Where("clients.email = ?", email).
  1163. Count(&attachmentCount).Error; err != nil {
  1164. continue
  1165. }
  1166. if attachmentCount > 0 {
  1167. continue
  1168. }
  1169. // "Ended the merge unattached" is true for a real remote deletion and
  1170. // equally true for a bad merge, so record a strike instead of deleting.
  1171. if err := markSyncOrphan(tx, email, now); err != nil {
  1172. logger.Warningf("setRemoteTraffic: mark orphan %q failed: %v", email, err)
  1173. continue
  1174. }
  1175. structuralChange = true
  1176. }
  1177. }
  1178. if err := clearSyncOrphanMarks(tx); err != nil {
  1179. logger.Warning("setRemoteTraffic: clear orphan marks failed:", err)
  1180. }
  1181. if err := liftActivatedClientRecordExpiries(tx); err != nil {
  1182. logger.Warning("setRemoteTraffic: lift activated expiries failed:", err)
  1183. }
  1184. if err := tx.Commit().Error; err != nil {
  1185. return false, err
  1186. }
  1187. committed = true
  1188. if lifecycleLifted && !dirty {
  1189. var already model.Node
  1190. if err := database.GetDB().Select("config_dirty").Where("id = ?", nodeID).First(&already).Error; err == nil && already.ConfigDirty {
  1191. logger.Debugf("setRemoteTraffic: node %d lifecycle lift; already dirty", nodeID)
  1192. } else {
  1193. logger.Infof("setRemoteTraffic: node %d lifecycle lift; marking dirty for re-push", nodeID)
  1194. if err := (&NodeService{}).MarkNodeDirty(nodeID); err != nil {
  1195. logger.Warningf("setRemoteTraffic: mark node %d dirty after lifecycle lift failed: %v", nodeID, err)
  1196. }
  1197. }
  1198. }
  1199. if len(adoptedInbounds) > 0 {
  1200. if mgr := runtime.GetManager(); mgr != nil {
  1201. if rt, rtErr := mgr.RuntimeFor(&nodeID); rtErr == nil {
  1202. if rem, ok := rt.(*runtime.Remote); ok {
  1203. for _, ib := range adoptedInbounds {
  1204. rem.RecordAdoptedInbound(ib)
  1205. }
  1206. }
  1207. }
  1208. }
  1209. }
  1210. if process := currentXrayProcess(); process != nil {
  1211. tree := snap.OnlineTree
  1212. switch {
  1213. case len(tree) == 0 && len(snap.OnlineEmails) > 0:
  1214. // Old-build node (no GUID tree): key its flat online list under its
  1215. // own effective identity so attribution still works for that branch.
  1216. tree = map[string][]string{selfKey: snap.OnlineEmails}
  1217. case guidShared && len(tree) > 0:
  1218. // Newer cloned node: its own clients arrive keyed under the shared
  1219. // panelGuid. Remap just that entry to the node-unique key so the
  1220. // clones don't merge; descendant subtrees keep their distinct GUIDs.
  1221. if _, ok := tree[nodeRow.Guid]; ok {
  1222. tree = remapGuidTreeKey(tree, nodeRow.Guid, selfKey)
  1223. }
  1224. }
  1225. process.SetNodeOnlineTree(nodeID, tree)
  1226. activeTree := normalizeActiveInboundTreeTags(snap.ActiveInboundTree, tagToCentral)
  1227. if guidShared && len(activeTree) > 0 {
  1228. if _, ok := activeTree[nodeRow.Guid]; ok {
  1229. activeTree = remapGuidTreeKey(activeTree, nodeRow.Guid, selfKey)
  1230. }
  1231. }
  1232. if len(activeTree) > 0 {
  1233. activeTree = filterGuidTreeKeys(activeTree, activeInboundGuidKeys(snap.Inbounds, tagToCentral, originGuidFor))
  1234. }
  1235. process.SetNodeActiveInboundTree(nodeID, activeTree)
  1236. }
  1237. return structuralChange, nil
  1238. }
  1239. func (s *InboundService) restartRemoteNodesOnDisable(nodeIDs []int) {
  1240. restartOnDisable, err := (&SettingService{}).GetRestartXrayOnClientDisable()
  1241. if err != nil {
  1242. logger.Warning("disableInvalidClients: get RestartXrayOnClientDisable failed:", err)
  1243. return
  1244. }
  1245. if !restartOnDisable {
  1246. return
  1247. }
  1248. for _, nodeID := range nodeIDs {
  1249. nodeIDCopy := nodeID
  1250. rt, rtErr := runtime.GetManager().RuntimeFor(&nodeIDCopy)
  1251. if rtErr != nil {
  1252. logger.Warning("disableInvalidClients: get runtime for node", nodeID, "failed:", rtErr)
  1253. continue
  1254. }
  1255. if rtErr = rt.RestartXray(context.Background()); rtErr != nil {
  1256. logger.Warning("disableInvalidClients: restart xray on node", nodeID, "failed:", rtErr)
  1257. }
  1258. }
  1259. }
  1260. func (s *InboundService) GetOnlineClients() []string {
  1261. process := currentXrayProcess()
  1262. if process == nil {
  1263. return []string{}
  1264. }
  1265. return process.GetOnlineClients()
  1266. }
  1267. // GetOnlineClientsByGuid returns online emails keyed by the panelGuid of the
  1268. // node that physically hosts each set: this panel's own clients under its own
  1269. // GUID, plus every node in the tree under its GUID (#4983). Replaces the old
  1270. // node-id keying so a client three hops down is attributed to its real node,
  1271. // not the intermediate one it was synced through.
  1272. func (s *InboundService) GetOnlineClientsByGuid() map[string][]string {
  1273. process := currentXrayProcess()
  1274. if process == nil {
  1275. return map[string][]string{}
  1276. }
  1277. out := process.GetMergedNodeTrees()
  1278. if local := process.GetLocalOnlineClients(); len(local) > 0 {
  1279. if guid := s.panelGuid(); guid != "" {
  1280. out[guid] = mergeEmails(out[guid], local)
  1281. }
  1282. }
  1283. return out
  1284. }
  1285. // GetActiveInboundsByGuid returns the inbound tags that carried traffic within
  1286. // the grace window, keyed by the panelGuid of the node that physically hosts
  1287. // each inbound. A GUID missing from the map means "don't gate" for that node's
  1288. // inbounds (old-build node or no active-inbound signal).
  1289. func (s *InboundService) GetActiveInboundsByGuid() map[string][]string {
  1290. process := currentXrayProcess()
  1291. if process == nil {
  1292. return map[string][]string{}
  1293. }
  1294. out := process.GetMergedActiveInboundTrees()
  1295. active := process.GetLocalActiveInbounds()
  1296. if len(active) == 0 {
  1297. return out
  1298. }
  1299. guid := s.panelGuid()
  1300. if guid == "" {
  1301. return out
  1302. }
  1303. out[guid] = mergeEmails(out[guid], active)
  1304. return out
  1305. }
  1306. func (s *InboundService) SetNodeOnlineTree(nodeID int, tree map[string][]string) {
  1307. if process := currentXrayProcess(); process != nil {
  1308. process.SetNodeOnlineTree(nodeID, tree)
  1309. }
  1310. }
  1311. func (s *InboundService) ClearNodeOnlineClients(nodeID int) {
  1312. if process := currentXrayProcess(); process != nil {
  1313. process.ClearNodeOnlineClients(nodeID)
  1314. }
  1315. }
  1316. // panelGuid returns this panel's stable self-identifier, used to key the local
  1317. // panel's own clients in the per-node online maps (#4983).
  1318. func (s *InboundService) panelGuid() string {
  1319. guid, _ := (&SettingService{}).GetPanelGuid()
  1320. return guid
  1321. }
  1322. // synthNodeGuid is the stable per-node fallback identity for a directly-attached
  1323. // node whose panel hasn't reported a panelGuid yet (old build). Node ids are
  1324. // master-local, so this only composes for direct nodes — exactly the pre-#4983
  1325. // flat-topology case where an old-build node appears.
  1326. func synthNodeGuid(nodeID int) string {
  1327. return fmt.Sprintf("node:%d", nodeID)
  1328. }
  1329. // mergeEmails returns the deduped union of two email slices.
  1330. func mergeEmails(a, b []string) []string {
  1331. if len(a) == 0 {
  1332. return b
  1333. }
  1334. seen := make(map[string]struct{}, len(a)+len(b))
  1335. out := make([]string, 0, len(a)+len(b))
  1336. for _, e := range a {
  1337. if _, ok := seen[e]; !ok {
  1338. seen[e] = struct{}{}
  1339. out = append(out, e)
  1340. }
  1341. }
  1342. for _, e := range b {
  1343. if _, ok := seen[e]; !ok {
  1344. seen[e] = struct{}{}
  1345. out = append(out, e)
  1346. }
  1347. }
  1348. return out
  1349. }
  1350. func remapGuidTreeKey(tree map[string][]string, from, to string) map[string][]string {
  1351. if from == "" || to == "" || from == to {
  1352. return tree
  1353. }
  1354. remapped := make(map[string][]string, len(tree))
  1355. for guid, values := range tree {
  1356. if guid == from {
  1357. guid = to
  1358. }
  1359. remapped[guid] = mergeEmails(remapped[guid], values)
  1360. }
  1361. return remapped
  1362. }
  1363. func normalizeActiveInboundTreeTags(tree map[string][]string, tagToCentral map[string]*model.Inbound) map[string][]string {
  1364. if len(tree) == 0 {
  1365. return nil
  1366. }
  1367. out := make(map[string][]string, len(tree))
  1368. for guid, tags := range tree {
  1369. if guid == "" || len(tags) == 0 {
  1370. continue
  1371. }
  1372. seen := make(map[string]struct{}, len(tags))
  1373. for _, tag := range tags {
  1374. if tag == "" {
  1375. continue
  1376. }
  1377. if central, ok := tagToCentral[tag]; ok && central != nil && central.Tag != "" {
  1378. tag = central.Tag
  1379. }
  1380. if _, dup := seen[tag]; dup {
  1381. continue
  1382. }
  1383. seen[tag] = struct{}{}
  1384. out[guid] = append(out[guid], tag)
  1385. }
  1386. }
  1387. if len(out) == 0 {
  1388. return nil
  1389. }
  1390. return out
  1391. }
  1392. func activeInboundGuidKeys(inbounds []*model.Inbound, tagToCentral map[string]*model.Inbound, originGuidFor func(*model.Inbound) string) map[string]struct{} {
  1393. allowed := make(map[string]struct{})
  1394. for _, ib := range inbounds {
  1395. if ib == nil {
  1396. continue
  1397. }
  1398. if _, ok := tagToCentral[ib.Tag]; !ok {
  1399. continue
  1400. }
  1401. if guid := originGuidFor(ib); guid != "" {
  1402. allowed[guid] = struct{}{}
  1403. }
  1404. }
  1405. return allowed
  1406. }
  1407. func filterGuidTreeKeys(tree map[string][]string, allowed map[string]struct{}) map[string][]string {
  1408. if len(tree) == 0 || len(allowed) == 0 {
  1409. return nil
  1410. }
  1411. out := make(map[string][]string, len(tree))
  1412. for guid, values := range tree {
  1413. if _, ok := allowed[guid]; !ok {
  1414. continue
  1415. }
  1416. if len(values) > 0 {
  1417. out[guid] = values
  1418. }
  1419. }
  1420. if len(out) == 0 {
  1421. return nil
  1422. }
  1423. return out
  1424. }
  1425. func (s *InboundService) GetClientsLastOnline() (map[string]int64, error) {
  1426. db := database.GetDB()
  1427. var rows []xray.ClientTraffic
  1428. err := db.Model(&xray.ClientTraffic{}).Select("email, last_online").Find(&rows).Error
  1429. if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
  1430. return nil, err
  1431. }
  1432. result := make(map[string]int64, len(rows))
  1433. for _, r := range rows {
  1434. result[r.Email] = r.LastOnline
  1435. }
  1436. return result, nil
  1437. }
  1438. // RefreshLocalOnlineClients folds the emails and inbound tags active on this
  1439. // panel's own xray this poll into the local online/active sets, applying the
  1440. // grace window and pruning stale entries. Pass nil to only prune. See
  1441. // xray.Process for why the local sets are kept separate from the shared
  1442. // last_online column.
  1443. func (s *InboundService) RefreshLocalOnlineClients(activeEmails, activeInboundTags []string) {
  1444. if process := currentXrayProcess(); process != nil {
  1445. process.RefreshLocalOnline(activeEmails, activeInboundTags, time.Now().UnixMilli(), onlineGracePeriodMs)
  1446. }
  1447. }
  1448. func (s *InboundService) FilterAndSortClientEmails(emails []string) ([]string, []string, error) {
  1449. db := database.GetDB()
  1450. // Step 1: Get ClientTraffic records for emails in the input list.
  1451. // Chunked to stay under SQLite's bind-variable limit on huge inputs.
  1452. uniqEmails := uniqueNonEmptyStrings(emails)
  1453. clients := make([]xray.ClientTraffic, 0, len(uniqEmails))
  1454. for _, batch := range chunkStrings(uniqEmails, sqliteMaxVars) {
  1455. var page []xray.ClientTraffic
  1456. if err := db.Where("email IN ?", batch).Find(&page).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
  1457. return nil, nil, err
  1458. }
  1459. clients = append(clients, page...)
  1460. }
  1461. // Step 2: Sort clients by (Up + Down) descending
  1462. sort.Slice(clients, func(i, j int) bool {
  1463. return (clients[i].Up + clients[i].Down) > (clients[j].Up + clients[j].Down)
  1464. })
  1465. // Step 3: Extract sorted valid emails and track found ones
  1466. validEmails := make([]string, 0, len(clients))
  1467. found := make(map[string]bool)
  1468. for _, client := range clients {
  1469. validEmails = append(validEmails, client.Email)
  1470. found[client.Email] = true
  1471. }
  1472. // Step 4: Identify emails that were not found in the database
  1473. extraEmails := make([]string, 0)
  1474. for _, email := range emails {
  1475. if !found[email] {
  1476. extraEmails = append(extraEmails, email)
  1477. }
  1478. }
  1479. return validEmails, extraEmails, nil
  1480. }