inbound_node.go 54 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635
  1. package service
  2. import (
  3. "context"
  4. "errors"
  5. "fmt"
  6. "sort"
  7. "strings"
  8. "sync"
  9. "time"
  10. "github.com/mhsanaei/3x-ui/v3/internal/database"
  11. "github.com/mhsanaei/3x-ui/v3/internal/database/model"
  12. "github.com/mhsanaei/3x-ui/v3/internal/logger"
  13. "github.com/mhsanaei/3x-ui/v3/internal/util/common"
  14. "github.com/mhsanaei/3x-ui/v3/internal/web/runtime"
  15. "github.com/mhsanaei/3x-ui/v3/internal/xray"
  16. "gorm.io/gorm"
  17. "gorm.io/gorm/clause"
  18. )
  19. var reportedRemoteTagConflict sync.Map
  20. var reportedForeignClientClaim sync.Map
  21. // nodeBulkPushThreshold caps how many per-client RPCs a single operation will
  22. // stream to a remote node. Above it, the panel marks the node dirty instead and
  23. // lets one ReconcileNode push converge the whole inbound — far cheaper than M
  24. // sequential round-trips. Small ops stay on the live per-client path.
  25. const nodeBulkPushThreshold = 32
  26. // nodeClientPushTimeout bounds the synchronous per-client push: the change is
  27. // committed and the node flagged dirty, so a slow node defers to the reconcile.
  28. const nodeClientPushTimeout = 4 * time.Second
  29. func nodePushContext() (context.Context, context.CancelFunc) {
  30. return context.WithTimeout(context.Background(), nodeClientPushTimeout)
  31. }
  32. func (s *InboundService) runtimeFor(ib *model.Inbound) (runtime.Runtime, error) {
  33. mgr := runtime.GetManager()
  34. if mgr == nil {
  35. return nil, fmt.Errorf("runtime manager not initialised")
  36. }
  37. return mgr.RuntimeFor(ib.NodeID)
  38. }
  39. func (s *InboundService) nodePushPlan(ib *model.Inbound) (runtime.Runtime, bool, bool, error) {
  40. if ib.NodeID == nil {
  41. rt, err := s.runtimeFor(ib)
  42. if err != nil {
  43. return nil, false, false, nil
  44. }
  45. return rt, true, false, nil
  46. }
  47. nodeSvc := NodeService{}
  48. enabled, status, _, _, err := nodeSvc.NodeSyncState(*ib.NodeID)
  49. if err != nil {
  50. return nil, false, false, err
  51. }
  52. if !enabled || status == "offline" {
  53. return nil, false, true, nil
  54. }
  55. rt, err := s.runtimeFor(ib)
  56. if err != nil {
  57. return nil, false, true, nil
  58. }
  59. return rt, true, false, nil
  60. }
  61. func (s *InboundService) NodeIsPending(nodeID *int) bool {
  62. if nodeID == nil {
  63. return false
  64. }
  65. return (&NodeService{}).IsNodePending(*nodeID)
  66. }
  67. func (s *InboundService) AnyNodePending(inboundIds []int) bool {
  68. if len(inboundIds) == 0 {
  69. return false
  70. }
  71. nodeSvc := NodeService{}
  72. for _, id := range inboundIds {
  73. ib, err := s.GetInbound(id)
  74. if err != nil || ib.NodeID == nil {
  75. continue
  76. }
  77. if nodeSvc.IsNodePending(*ib.NodeID) {
  78. return true
  79. }
  80. }
  81. return false
  82. }
  83. // ReconcileNode pushes every inbound and sweeps undesired remote tags even when
  84. // individual operations fail, returning the failures joined: one inbound the
  85. // node rejects (e.g. a legacy protocol failing validation, #5685) must not
  86. // stall the rest of the node's config — or, via syncOne, its traffic sync.
  87. func (s *InboundService) ReconcileNode(ctx context.Context, rt *runtime.Remote, n *model.Node) error {
  88. if rt == nil || n == nil || n.Id <= 0 {
  89. return nil
  90. }
  91. nodeID := n.Id
  92. db := database.GetDB()
  93. var inbounds []*model.Inbound
  94. if err := db.Model(model.Inbound{}).Where("node_id = ?", nodeID).Find(&inbounds).Error; err != nil {
  95. return err
  96. }
  97. remoteInbounds, err := rt.ListInboundOptions(ctx)
  98. if err != nil {
  99. return err
  100. }
  101. remoteTags := make([]string, 0, len(remoteInbounds))
  102. remoteTagSet := make(map[string]struct{}, len(remoteTags))
  103. for _, remoteIb := range remoteInbounds {
  104. remoteTags = append(remoteTags, remoteIb.Tag)
  105. remoteTagSet[remoteIb.Tag] = struct{}{}
  106. }
  107. prefix := nodeTagPrefix(&nodeID)
  108. desiredTags := make(map[string]struct{}, len(inbounds)*2)
  109. var errs []error
  110. for _, ib := range inbounds {
  111. desiredTags[ib.Tag] = struct{}{}
  112. // existsOnNode: does the node already report this inbound under any of the
  113. // tag forms it may be stored as? If so, an unchanged push can be skipped.
  114. _, existsOnNode := remoteTagSet[ib.Tag]
  115. if prefix != "" {
  116. if stripped, found := strings.CutPrefix(ib.Tag, prefix); found {
  117. desiredTags[stripped] = struct{}{}
  118. if _, ok := remoteTagSet[stripped]; ok {
  119. existsOnNode = true
  120. }
  121. } else {
  122. desiredTags[prefix+ib.Tag] = struct{}{}
  123. if _, ok := remoteTagSet[prefix+ib.Tag]; ok {
  124. existsOnNode = true
  125. }
  126. }
  127. }
  128. runtimeIb := ib
  129. if built, bErr := s.buildInboundForNodePush(db, ib); bErr == nil {
  130. runtimeIb = built
  131. }
  132. if !existsOnNode && n.Guid != "" && ib.OriginNodeGuid == n.Guid {
  133. var compatible []runtime.RemoteInboundOption
  134. for _, remoteIb := range remoteInbounds {
  135. if remoteIb.Port == runtimeIb.Port &&
  136. remoteIb.Protocol == runtimeIb.Protocol &&
  137. strings.TrimSpace(remoteIb.Listen) == strings.TrimSpace(runtimeIb.Listen) {
  138. compatible = append(compatible, remoteIb)
  139. }
  140. }
  141. switch len(compatible) {
  142. case 1:
  143. alias := compatible[0]
  144. desiredTags[alias.Tag] = struct{}{}
  145. rt.AdoptInboundAlias(runtimeIb, alias)
  146. existsOnNode = true
  147. logger.Infof("adopted compatible inbound %q on node %s as %q", alias.Tag, n.Name, ib.Tag)
  148. case 0:
  149. // No compatible occupant: keep the normal create path, which
  150. // leaves a real port/protocol drift loud.
  151. default:
  152. for _, candidate := range compatible {
  153. desiredTags[candidate.Tag] = struct{}{}
  154. }
  155. errs = append(errs, fmt.Errorf("reconcile inbound %q: ambiguous compatible remote inbounds", ib.Tag))
  156. continue
  157. }
  158. }
  159. if _, err := rt.ReconcileInbound(ctx, runtimeIb, existsOnNode); err != nil {
  160. errs = append(errs, fmt.Errorf("reconcile inbound %q: %w", ib.Tag, err))
  161. }
  162. }
  163. // Before the next clean sync adopts the node's inbounds, "absent locally"
  164. // means "not imported yet" — sweeping now would wipe the node at onboarding.
  165. if n.InboundsAdoptedAt == 0 {
  166. return errors.Join(errs...)
  167. }
  168. // In "selected" sync mode the panel only manages the selected tags: the
  169. // rest were never imported, so their absence from the local DB must not
  170. // delete them from the node. Only a selected tag missing locally (the
  171. // panel deleted it while the node was unreachable) may be swept.
  172. selected := nodeSelectedTagSet(n)
  173. for _, tag := range remoteTags {
  174. if _, want := desiredTags[tag]; want {
  175. continue
  176. }
  177. if selected != nil {
  178. if _, managed := selected[tag]; !managed {
  179. continue
  180. }
  181. }
  182. if err := rt.DelInbound(ctx, &model.Inbound{Tag: tag}); err != nil {
  183. errs = append(errs, fmt.Errorf("reconcile delete %q: %w", tag, err))
  184. }
  185. }
  186. return errors.Join(errs...)
  187. }
  188. const resetGracePeriodMs int64 = 30000
  189. // onlineGracePeriodMs must comfortably exceed the 5s traffic-poll interval —
  190. // Xray's stats counters often report a zero delta for an active session across
  191. // a single poll, so a 5s grace would still drop the client on the next tick.
  192. // ~4 polls of slack keeps idle-but-connected clients visible without lingering
  193. // long after a real disconnect.
  194. const onlineGracePeriodMs int64 = 20000
  195. type nodeTrafficCounter struct {
  196. Up int64
  197. Down int64
  198. }
  199. func (s *InboundService) upsertNodeBaseline(tx *gorm.DB, nodeID int, email string, up, down int64) error {
  200. return tx.Clauses(clause.OnConflict{
  201. Columns: []clause.Column{{Name: "node_id"}, {Name: "email"}},
  202. DoUpdates: clause.AssignmentColumns([]string{"up", "down"}),
  203. }).Create(&model.NodeClientTraffic{NodeId: nodeID, Email: email, Up: up, Down: down}).Error
  204. }
  205. // mergeActivationExpiry: master absolute wins; node may only activate when
  206. // master is unset/duration. Node auto-renew goes through nodeClientRenewed.
  207. func mergeActivationExpiry(existing, node int64) int64 {
  208. if existing > 0 {
  209. return existing
  210. }
  211. return node
  212. }
  213. // masterLimitsAllowClient reports whether the master's own deadline and quota
  214. // (including this tick's deltas) still permit the client.
  215. func masterLimitsAllowClient(master *xray.ClientTraffic, now, deltaUp, deltaDown int64) bool {
  216. if master == nil {
  217. return false
  218. }
  219. if master.ExpiryTime > 0 && master.ExpiryTime <= now {
  220. return false
  221. }
  222. if master.Total > 0 && master.Up+deltaUp+master.Down+deltaDown >= master.Total {
  223. return false
  224. }
  225. return true
  226. }
  227. // nodeDisableIsStale reports an enable=false the node decided against limits the
  228. // master has since changed, so it must not latch back (#6228 / #4917).
  229. func nodeDisableIsStale(master *xray.ClientTraffic, node xray.ClientTraffic, now, deltaUp, deltaDown int64) bool {
  230. if master == nil {
  231. return false
  232. }
  233. // Matching limits mean the node judged the client on the master's own terms:
  234. // that verdict is genuine and still latches, as #4917 requires.
  235. if node.ExpiryTime == master.ExpiryTime && node.Total == master.Total {
  236. return false
  237. }
  238. return masterLimitsAllowClient(master, now, deltaUp, deltaDown)
  239. }
  240. func clampTrafficCounter(v int64) int64 {
  241. if v > database.TrafficMax {
  242. return database.TrafficMax
  243. }
  244. if v < 0 {
  245. return 0
  246. }
  247. return v
  248. }
  249. // applyMasterClientLifecycle overlays the already-merged master row onto a
  250. // node-reported client for SyncInbound (#6228).
  251. func applyMasterClientLifecycle(c *model.Client, master *xray.ClientTraffic, cs *xray.ClientTraffic) {
  252. if master == nil {
  253. // No central row to speak for the client: the node's own latch is all
  254. // there is, and it may only disable.
  255. if cs != nil && !cs.Enable {
  256. c.Enable = false
  257. }
  258. return
  259. }
  260. c.ExpiryTime = mergeActivationExpiry(master.ExpiryTime, c.ExpiryTime)
  261. c.Enable = master.Enable
  262. }
  263. // nodeClientRenewed reports a node-side auto-renew: an absolute deadline moved
  264. // forward, evidenced by a renewal-count bump or a drop below the stored baseline.
  265. func nodeClientRenewed(existing *xray.ClientTraffic, cs xray.ClientTraffic, canon, base nodeTrafficCounter) bool {
  266. if (cs.Reset <= 0 && cs.ResetDay <= 0) || cs.ExpiryTime <= 0 || existing.ExpiryTime <= 0 {
  267. return false
  268. }
  269. if cs.ExpiryTime <= existing.ExpiryTime {
  270. return false
  271. }
  272. // A client that used no traffic in the period never dips, so the renewal
  273. // counter is the only evidence autoRenewClients leaves behind (#6228).
  274. if cs.ResetCount > existing.ResetCount {
  275. return true
  276. }
  277. return canon.Up < base.Up || canon.Down < base.Down
  278. }
  279. // liftActivatedClientRecordExpiries copies a node-activated deadline from
  280. // client_traffics onto client records still holding the negative duration (#5714).
  281. func liftActivatedClientRecordExpiries(tx *gorm.DB) error {
  282. return tx.Exec(
  283. `UPDATE clients
  284. SET expiry_time = (SELECT ct.expiry_time FROM client_traffics ct WHERE ct.email = clients.email AND ct.expiry_time > 0 LIMIT 1)
  285. WHERE clients.expiry_time < 0
  286. AND EXISTS (SELECT 1 FROM client_traffics ct WHERE ct.email = clients.email AND ct.expiry_time > 0)`,
  287. ).Error
  288. }
  289. // SnapshotHasUnadoptedInbounds reports whether the snapshot carries a tag with
  290. // no central row yet, i.e. the next merge would adopt a new inbound.
  291. func (s *InboundService) SnapshotHasUnadoptedInbounds(nodeID int, snap *runtime.TrafficSnapshot) (bool, error) {
  292. if snap == nil || len(snap.Inbounds) == 0 {
  293. return false, nil
  294. }
  295. var tags []string
  296. if err := database.GetDB().Model(model.Inbound{}).
  297. Where("node_id = ?", nodeID).
  298. Pluck("tag", &tags).Error; err != nil {
  299. return false, err
  300. }
  301. prefix := nodeTagPrefix(&nodeID)
  302. known := make(map[string]struct{}, len(tags)*2)
  303. for _, tag := range tags {
  304. known[tag] = struct{}{}
  305. if prefix != "" {
  306. if stripped, found := strings.CutPrefix(tag, prefix); found {
  307. known[stripped] = struct{}{}
  308. } else {
  309. known[prefix+tag] = struct{}{}
  310. }
  311. }
  312. }
  313. for _, ib := range snap.Inbounds {
  314. if ib == nil {
  315. continue
  316. }
  317. if _, ok := known[ib.Tag]; !ok {
  318. return true, nil
  319. }
  320. }
  321. return false, nil
  322. }
  323. // SetRemoteTraffic merges a node snapshot. justPushed marks the tick whose
  324. // config push just landed, whose snapshot may still predate it (#6228).
  325. func (s *InboundService) SetRemoteTraffic(nodeID int, snap *runtime.TrafficSnapshot, dirty, justPushed bool) (bool, error) {
  326. var structuralChange bool
  327. err := submitTrafficWrite(func() error {
  328. var inner error
  329. structuralChange, inner = s.setRemoteTrafficLocked(nodeID, snap, dirty, justPushed)
  330. return inner
  331. })
  332. return structuralChange, err
  333. }
  334. // GetNodeInboundTrafficTotals returns the current cumulative up/down for every
  335. // node-hosted inbound, keyed by tag. The node sync diffs successive snapshots of
  336. // this to derive per-inbound speed for the dashboard — node inbounds have no
  337. // local Xray poll to produce live deltas the way local inbounds do.
  338. func (s *InboundService) GetNodeInboundTrafficTotals() (map[string][2]int64, error) {
  339. var rows []struct {
  340. Tag string
  341. Up int64
  342. Down int64
  343. }
  344. if err := database.GetDB().Table("inbounds").
  345. Select("tag, up, down").
  346. Where("node_id IS NOT NULL").
  347. Scan(&rows).Error; err != nil {
  348. return nil, err
  349. }
  350. out := make(map[string][2]int64, len(rows))
  351. for _, r := range rows {
  352. out[r.Tag] = [2]int64{r.Up, r.Down}
  353. }
  354. return out, nil
  355. }
  356. func adoptedWireChanged(c, snapIb *model.Inbound, adoptedSettings string) bool {
  357. return c.Settings != adoptedSettings ||
  358. c.Enable != snapIb.Enable ||
  359. c.Remark != snapIb.Remark ||
  360. c.SubSortIndex != normalizeSubSortIndex(snapIb.SubSortIndex) ||
  361. c.Listen != snapIb.Listen ||
  362. c.Port != snapIb.Port ||
  363. c.Protocol != snapIb.Protocol ||
  364. c.Total != snapIb.Total ||
  365. c.ExpiryTime != snapIb.ExpiryTime ||
  366. c.StreamSettings != snapIb.StreamSettings ||
  367. c.Sniffing != snapIb.Sniffing ||
  368. c.TrafficReset != snapIb.TrafficReset ||
  369. c.TrafficResetDay != normalizeTrafficResetDay(snapIb.TrafficResetDay)
  370. }
  371. // adoptedWireInbound is the central inbound as it reads after adopting the
  372. // node-reported wire fields — the payload the reconcile fingerprint must track.
  373. func adoptedWireInbound(c, snapIb *model.Inbound, adoptedSettings string) *model.Inbound {
  374. a := *c
  375. a.Enable = snapIb.Enable
  376. a.Remark = snapIb.Remark
  377. a.SubSortIndex = normalizeSubSortIndex(snapIb.SubSortIndex)
  378. a.Listen = snapIb.Listen
  379. a.Port = snapIb.Port
  380. a.Protocol = snapIb.Protocol
  381. a.Total = snapIb.Total
  382. a.ExpiryTime = snapIb.ExpiryTime
  383. a.Settings = adoptedSettings
  384. a.StreamSettings = snapIb.StreamSettings
  385. a.Sniffing = snapIb.Sniffing
  386. a.TrafficReset = snapIb.TrafficReset
  387. a.TrafficResetDay = normalizeTrafficResetDay(snapIb.TrafficResetDay)
  388. return &a
  389. }
  390. // clientEmailsOwnedElsewhere returns the emails attached only to inbounds of
  391. // other nodes: email is unique, so adopting one would overwrite a client this
  392. // node does not serve. Attached nowhere means soft-orphaned, hence adoptable.
  393. func clientEmailsOwnedElsewhere(tx *gorm.DB, nodeID int, emails []string) (map[string]struct{}, error) {
  394. attachedEmails := func(nodeScoped bool) ([]string, error) {
  395. q := tx.Table("clients").
  396. Joins("JOIN client_inbounds ON client_inbounds.client_id = clients.id").
  397. Joins("JOIN inbounds ON inbounds.id = client_inbounds.inbound_id").
  398. Where("clients.email IN ?", emails)
  399. if nodeScoped {
  400. q = q.Where("inbounds.node_id = ?", nodeID)
  401. }
  402. var rows []string
  403. err := q.Pluck("clients.email", &rows).Error
  404. return rows, err
  405. }
  406. attached, err := attachedEmails(false)
  407. if err != nil {
  408. return nil, err
  409. }
  410. if len(attached) == 0 {
  411. return nil, nil
  412. }
  413. owned, err := attachedEmails(true)
  414. if err != nil {
  415. return nil, err
  416. }
  417. ownedSet := make(map[string]struct{}, len(owned))
  418. for _, email := range owned {
  419. ownedSet[email] = struct{}{}
  420. }
  421. foreign := make(map[string]struct{})
  422. for _, email := range attached {
  423. if _, ok := ownedSet[email]; !ok {
  424. foreign[email] = struct{}{}
  425. }
  426. }
  427. return foreign, nil
  428. }
  429. func (s *InboundService) setRemoteTrafficLocked(nodeID int, snap *runtime.TrafficSnapshot, dirty, justPushed bool) (bool, error) {
  430. if snap == nil || nodeID <= 0 {
  431. return false, nil
  432. }
  433. db := database.GetDB()
  434. now := time.Now().UnixMilli()
  435. // originGuidFor attributes a synced inbound to the panel that physically
  436. // hosts it. A node's OWN inbounds report either an empty origin or — on
  437. // builds that set it locally — the node's own panelGuid; both resolve to
  438. // selfKey, which is the node's panelGuid unless that GUID is ambiguous
  439. // (shared with another node or the master, i.e. a cloned server), in which
  440. // case it falls back to the node-unique id so #4983 attribution doesn't
  441. // collapse two physical nodes into one bucket. Only a DIFFERENT, non-empty
  442. // origin (an inbound the node forwards from its own sub-node) is kept as-is,
  443. // so a chained Node1->Node2->Node3 still attributes Node3's inbounds to Node3.
  444. var nodeRow model.Node
  445. db.Select("guid", "config_dirty", "inbound_sync_mode", "inbound_tags").Where("id = ?", nodeID).First(&nodeRow)
  446. // Re-read inside the serialized writer: a client added while this snapshot
  447. // was in flight marks the node dirty after the caller sampled the flag.
  448. dirty = dirty || nodeRow.ConfigDirty
  449. // Adoption, record sync and sweeps still run on a just-pushed tick; only the
  450. // client lifecycle merge waits for a snapshot that reflects the push.
  451. lifecycleFrozen := dirty || justPushed
  452. nodeRow.Id = nodeID
  453. unmanagedTag := unmanagedTagPredicate(&nodeRow)
  454. selfKey := effectiveNodeKey(&model.Node{Id: nodeID, Guid: nodeRow.Guid})
  455. guidShared := nodeRow.Guid != "" && selfKey != nodeRow.Guid
  456. originGuidFor := func(snapIb *model.Inbound) string {
  457. if snapIb.OriginNodeGuid != "" && snapIb.OriginNodeGuid != nodeRow.Guid {
  458. return snapIb.OriginNodeGuid
  459. }
  460. return selfKey
  461. }
  462. var central []model.Inbound
  463. if err := db.Model(model.Inbound{}).
  464. Where("node_id = ?", nodeID).
  465. Find(&central).Error; err != nil {
  466. return false, err
  467. }
  468. // Index under the stored tag and its prefix-flipped form so a snap matches
  469. // whether the n<id>- prefix lives on the node side, the central side, or
  470. // neither — a mismatch must never spawn a duplicate central inbound.
  471. tagToCentral := make(map[string]*model.Inbound, len(central)*2)
  472. prefix := nodeTagPrefix(&nodeID)
  473. for i := range central {
  474. tagToCentral[central[i].Tag] = &central[i]
  475. if prefix != "" {
  476. if stripped, found := strings.CutPrefix(central[i].Tag, prefix); found {
  477. tagToCentral[stripped] = &central[i]
  478. } else {
  479. tagToCentral[prefix+central[i].Tag] = &central[i]
  480. }
  481. }
  482. }
  483. var centralClientStats []xray.ClientTraffic
  484. if len(central) > 0 {
  485. ids := make([]int, 0, len(central))
  486. for i := range central {
  487. ids = append(ids, central[i].Id)
  488. }
  489. if err := db.Model(xray.ClientTraffic{}).
  490. Where("inbound_id IN ?", ids).
  491. Find(&centralClientStats).Error; err != nil {
  492. return false, err
  493. }
  494. }
  495. type csKey struct {
  496. inboundID int
  497. email string
  498. }
  499. centralCS := make(map[csKey]*xray.ClientTraffic, len(centralClientStats))
  500. centralCSByEmail := make(map[string]*xray.ClientTraffic, len(centralClientStats))
  501. for i := range centralClientStats {
  502. centralCS[csKey{centralClientStats[i].InboundId, centralClientStats[i].Email}] = &centralClientStats[i]
  503. centralCSByEmail[centralClientStats[i].Email] = &centralClientStats[i]
  504. }
  505. nodeBaselines := make(map[string]nodeTrafficCounter)
  506. var baselineRows []model.NodeClientTraffic
  507. if err := db.Model(&model.NodeClientTraffic{}).
  508. Where("node_id = ?", nodeID).
  509. Find(&baselineRows).Error; err != nil {
  510. return false, err
  511. }
  512. for i := range baselineRows {
  513. nodeBaselines[baselineRows[i].Email] = nodeTrafficCounter{Up: baselineRows[i].Up, Down: baselineRows[i].Down}
  514. }
  515. var defaultUserId int
  516. if len(central) > 0 {
  517. defaultUserId = central[0].UserId
  518. } else {
  519. var u model.User
  520. if err := db.Model(model.User{}).Order("id asc").First(&u).Error; err == nil {
  521. defaultUserId = u.Id
  522. } else {
  523. defaultUserId = 1
  524. }
  525. }
  526. // Union of every email the snapshot still reports, across all inbounds.
  527. // The (node, email) baseline rows are keyed per node, not per inbound, so
  528. // the sweeps below must only drop one when the email left the node
  529. // entirely — an email whose stats moved to (or always lived under) a
  530. // sibling inbound still needs its baseline for the sibling's delta
  531. // computation (#5202).
  532. //
  533. // Xray counts traffic per email, not per inbound, so a multi-attached
  534. // client's shared counter is copied onto every inbound it's on. Fold each
  535. // email to its per-field max (nodeEmailTotals) so divergent copies can't make
  536. // the reset clamp re-add a lower sibling as fresh traffic (#5274).
  537. snapEmailsAll := make(map[string]struct{})
  538. nodeEmailTotals := make(map[string]nodeTrafficCounter)
  539. for _, snapIb := range snap.Inbounds {
  540. if snapIb == nil {
  541. continue
  542. }
  543. for i := range snapIb.ClientStats {
  544. email := snapIb.ClientStats[i].Email
  545. snapEmailsAll[email] = struct{}{}
  546. cur := nodeEmailTotals[email]
  547. if snapIb.ClientStats[i].Up > cur.Up {
  548. cur.Up = snapIb.ClientStats[i].Up
  549. }
  550. if snapIb.ClientStats[i].Down > cur.Down {
  551. cur.Down = snapIb.ClientStats[i].Down
  552. }
  553. nodeEmailTotals[email] = cur
  554. }
  555. }
  556. // Membership set for the rowExists checks below. Only the snapshot's emails
  557. // are ever probed, so scope the lookup to those instead of plucking the whole
  558. // client_traffics table (50k+ rows) on every node poll.
  559. existingEmails := make(map[string]struct{}, len(snapEmailsAll))
  560. if len(snapEmailsAll) > 0 {
  561. snapEmailList := make([]string, 0, len(snapEmailsAll))
  562. for email := range snapEmailsAll {
  563. snapEmailList = append(snapEmailList, email)
  564. }
  565. for _, batch := range chunkStrings(snapEmailList, sqliteMaxVars) {
  566. var found []string
  567. if err := db.Model(xray.ClientTraffic{}).Where("email IN ?", batch).Pluck("email", &found).Error; err != nil {
  568. return false, err
  569. }
  570. for _, e := range found {
  571. existingEmails[e] = struct{}{}
  572. }
  573. }
  574. }
  575. tx := db.Begin()
  576. committed := false
  577. defer func() {
  578. if !committed {
  579. tx.Rollback()
  580. }
  581. }()
  582. structuralChange := false
  583. lifecycleLifted := false
  584. var adoptedInbounds []*model.Inbound
  585. type pendingAdopt struct {
  586. central *model.Inbound
  587. snapIb *model.Inbound
  588. wireSettings string
  589. }
  590. var pendingAdopts []pendingAdopt
  591. newInboundIDs := make(map[int]struct{})
  592. snapTags := make(map[string]struct{}, len(snap.Inbounds))
  593. for _, snapIb := range snap.Inbounds {
  594. if snapIb == nil {
  595. continue
  596. }
  597. snapTags[snapIb.Tag] = struct{}{}
  598. // Record the prefix-flipped form too so the orphan sweep below keeps a
  599. // central inbound whether its tag carries the n<id>- prefix or not.
  600. if prefix != "" {
  601. if stripped, found := strings.CutPrefix(snapIb.Tag, prefix); found {
  602. snapTags[stripped] = struct{}{}
  603. } else {
  604. snapTags[prefix+snapIb.Tag] = struct{}{}
  605. }
  606. }
  607. c, ok := tagToCentral[snapIb.Tag]
  608. if !ok {
  609. origin := originGuidFor(snapIb)
  610. var compatible []*model.Inbound
  611. for i := range central {
  612. candidate := &central[i]
  613. if candidate.OriginNodeGuid == origin &&
  614. candidate.Port == snapIb.Port &&
  615. candidate.Protocol == snapIb.Protocol &&
  616. strings.TrimSpace(candidate.Listen) == strings.TrimSpace(snapIb.Listen) {
  617. compatible = append(compatible, candidate)
  618. }
  619. }
  620. switch len(compatible) {
  621. case 1:
  622. c, ok = compatible[0], true
  623. tagToCentral[snapIb.Tag] = c
  624. snapTags[c.Tag] = struct{}{}
  625. case 0:
  626. // A genuinely new inbound follows the normal adoption path.
  627. default:
  628. return false, fmt.Errorf("setRemoteTraffic: inbound %q has ambiguous compatible central aliases", snapIb.Tag)
  629. }
  630. }
  631. if !ok {
  632. if dirty {
  633. continue
  634. }
  635. // Try snap.Tag first; on collision fall back to the n<id>-
  636. // prefixed form so local+node can both own the same port.
  637. pickFreeTag := func() (string, error) {
  638. candidates := []string{snapIb.Tag}
  639. if prefix != "" && !strings.HasPrefix(snapIb.Tag, prefix) {
  640. candidates = append(candidates, prefix+snapIb.Tag)
  641. }
  642. for _, t := range candidates {
  643. var owner model.Inbound
  644. err := tx.Where("tag = ?", t).First(&owner).Error
  645. if errors.Is(err, gorm.ErrRecordNotFound) {
  646. return t, nil
  647. }
  648. if err != nil {
  649. return "", err
  650. }
  651. }
  652. return "", nil
  653. }
  654. chosenTag, err := pickFreeTag()
  655. if err != nil {
  656. logger.Warningf("setRemoteTraffic: check tag %q failed: %v", snapIb.Tag, err)
  657. continue
  658. }
  659. if chosenTag == "" {
  660. key := fmt.Sprintf("%d:%s", nodeID, snapIb.Tag)
  661. if _, seen := reportedRemoteTagConflict.LoadOrStore(key, struct{}{}); !seen {
  662. logger.Warningf(
  663. "setRemoteTraffic: tag %q from node %d collides with an existing inbound even after the n%d- prefix — skipping (rename one side to remove the duplicate)",
  664. snapIb.Tag, nodeID, nodeID,
  665. )
  666. }
  667. continue
  668. }
  669. reportedRemoteTagConflict.Delete(fmt.Sprintf("%d:%s", nodeID, snapIb.Tag))
  670. newIb := model.Inbound{
  671. UserId: defaultUserId,
  672. NodeID: &nodeID,
  673. OriginNodeGuid: originGuidFor(snapIb),
  674. Tag: chosenTag,
  675. Listen: snapIb.Listen,
  676. Port: snapIb.Port,
  677. Protocol: snapIb.Protocol,
  678. Settings: snapIb.Settings,
  679. StreamSettings: snapIb.StreamSettings,
  680. Sniffing: snapIb.Sniffing,
  681. TrafficReset: snapIb.TrafficReset,
  682. TrafficResetDay: normalizeTrafficResetDay(snapIb.TrafficResetDay),
  683. LastTrafficResetTime: snapIb.LastTrafficResetTime,
  684. Enable: snapIb.Enable,
  685. Remark: snapIb.Remark,
  686. SubSortIndex: normalizeSubSortIndex(snapIb.SubSortIndex),
  687. Total: snapIb.Total,
  688. ExpiryTime: snapIb.ExpiryTime,
  689. Up: snapIb.Up,
  690. Down: snapIb.Down,
  691. ShareAddrStrategy: "node",
  692. DisableFlow: snapIb.DisableFlow,
  693. }
  694. if err := tx.Create(&newIb).Error; err != nil {
  695. logger.Warningf("setRemoteTraffic: create central inbound for tag %q failed: %v", snapIb.Tag, err)
  696. continue
  697. }
  698. tagToCentral[snapIb.Tag] = &newIb
  699. if newIb.Tag != snapIb.Tag {
  700. tagToCentral[newIb.Tag] = &newIb
  701. }
  702. if rows := adoptedHostRows(snap.HostGroups, snapIb.Id, newIb.Id); len(rows) > 0 {
  703. if err := tx.Create(&rows).Error; err != nil {
  704. logger.Warningf("setRemoteTraffic: adopt host rows for tag %q failed: %v", newIb.Tag, err)
  705. }
  706. }
  707. newInboundIDs[newIb.Id] = struct{}{}
  708. structuralChange = true
  709. continue
  710. }
  711. inGrace := c.LastTrafficResetTime > 0 && now-c.LastTrafficResetTime < resetGracePeriodMs
  712. // Adopting the node's settings verbatim would re-add a client the master
  713. // deleted moments ago if this snapshot was fetched before the deletion
  714. // push landed — filter just-deleted emails out while their tombstone lives.
  715. adoptedSettings := snapIb.Settings
  716. if stripped, changed := stripTombstonedClients(adoptedSettings); changed {
  717. adoptedSettings = stripped
  718. }
  719. if deduped, changed := dedupeSettingsClients(adoptedSettings); changed {
  720. adoptedSettings = deduped
  721. }
  722. updates := map[string]any{}
  723. if !dirty {
  724. // Defer lifecycle lift until after client_traffics absorbs this tick's
  725. // deltas so quota stale-disable matches SQL (#6228).
  726. pendingAdopts = append(pendingAdopts, pendingAdopt{
  727. central: c, snapIb: snapIb, wireSettings: adoptedSettings,
  728. })
  729. updates["enable"] = snapIb.Enable
  730. updates["remark"] = snapIb.Remark
  731. updates["sub_sort_index"] = normalizeSubSortIndex(snapIb.SubSortIndex)
  732. updates["listen"] = snapIb.Listen
  733. updates["port"] = snapIb.Port
  734. updates["protocol"] = snapIb.Protocol
  735. updates["total"] = snapIb.Total
  736. updates["expiry_time"] = snapIb.ExpiryTime
  737. updates["stream_settings"] = snapIb.StreamSettings
  738. updates["sniffing"] = snapIb.Sniffing
  739. updates["traffic_reset"] = snapIb.TrafficReset
  740. updates["traffic_reset_day"] = normalizeTrafficResetDay(snapIb.TrafficResetDay)
  741. updates["last_traffic_reset_time"] = snapIb.LastTrafficResetTime
  742. }
  743. if !inGrace || (snapIb.Up+snapIb.Down) <= (c.Up+c.Down) {
  744. updates["up"] = snapIb.Up
  745. updates["down"] = snapIb.Down
  746. }
  747. // Physical-home attribution is independent of config-dirty state, so
  748. // keep it current even while the node has pending offline edits. Writes
  749. // once to backfill an existing row, then stays equal (#4983).
  750. if og := originGuidFor(snapIb); c.OriginNodeGuid != og {
  751. updates["origin_node_guid"] = og
  752. }
  753. if !dirty && (c.Remark != snapIb.Remark ||
  754. c.Listen != snapIb.Listen ||
  755. c.Port != snapIb.Port ||
  756. c.Total != snapIb.Total ||
  757. c.ExpiryTime != snapIb.ExpiryTime ||
  758. c.Enable != snapIb.Enable) {
  759. structuralChange = true
  760. }
  761. if len(updates) > 0 {
  762. if err := tx.Model(model.Inbound{}).
  763. Where("id = ?", c.Id).
  764. Updates(updates).Error; err != nil {
  765. return false, err
  766. }
  767. }
  768. }
  769. for _, c := range central {
  770. if dirty {
  771. continue
  772. }
  773. // A node inbound created disabled is never delivered, so its absence from
  774. // the snapshot is ambiguous rather than evidence of a node-side delete.
  775. if !c.Enable {
  776. continue
  777. }
  778. if len(snapTags) == 0 {
  779. // A node mid-restart or with a transient DB error can return an empty
  780. // inbound list with success=true. Treat "zero inbounds reported" as
  781. // "nothing to say", not "delete all my inbounds" — otherwise a blip
  782. // wipes the node's central inbounds and every client on them (and
  783. // resets traffic history on re-create). A real per-inbound deletion
  784. // still sweeps, because the node keeps reporting its other inbounds.
  785. continue
  786. }
  787. if _, kept := snapTags[c.Tag]; kept {
  788. continue
  789. }
  790. if unmanagedTag(c.Tag) {
  791. continue
  792. }
  793. // This drops the central inbound and its clients' traffic history, so say
  794. // so: silent removal is indistinguishable from an inbound never arriving.
  795. logger.Warningf("setRemoteTraffic: node %d no longer reports inbound %q (id %d, port %d) — removing it centrally", nodeID, c.Tag, c.Id, c.Port)
  796. var goneEmails []string
  797. if err := tx.Model(xray.ClientTraffic{}).
  798. Where("inbound_id = ?", c.Id).
  799. Pluck("email", &goneEmails).Error; err != nil {
  800. return false, err
  801. }
  802. if len(goneEmails) > 0 {
  803. // Baselines are per (node, email), not per inbound: keep them for
  804. // emails the snapshot still reports under a sibling inbound (#5202).
  805. baselineGone := make([]string, 0, len(goneEmails))
  806. for _, e := range goneEmails {
  807. if _, still := snapEmailsAll[e]; !still {
  808. baselineGone = append(baselineGone, e)
  809. }
  810. }
  811. // Chunk to avoid SQLite bind var limit when a node has many clients
  812. // removed (e.g. after API bulk delete or structural change on node inbound).
  813. for _, batch := range chunkStrings(baselineGone, sqliteMaxVars) {
  814. if err := tx.Where("node_id = ? AND email IN ?", nodeID, batch).
  815. Delete(&model.NodeClientTraffic{}).Error; err != nil {
  816. return false, err
  817. }
  818. }
  819. // The per-email row is the shared accumulator across every inbound
  820. // (and node) the email is attached to. Only drop it when this was the
  821. // email's last inbound — wiping it while a sibling still feeds it
  822. // loses the summed history, and the next node sync would re-seed the
  823. // row with that node's counter alone.
  824. sharedEmails, sErr := s.emailsUsedByOtherInbounds(goneEmails, c.Id)
  825. if sErr != nil {
  826. return false, sErr
  827. }
  828. delEmails := make([]string, 0, len(goneEmails))
  829. for _, e := range goneEmails {
  830. if !sharedEmails[strings.ToLower(strings.TrimSpace(e))] {
  831. delEmails = append(delEmails, e)
  832. }
  833. }
  834. for _, batch := range chunkStrings(delEmails, sqliteMaxVars) {
  835. if err := tx.Where("inbound_id = ? AND email IN ?", c.Id, batch).
  836. Delete(&xray.ClientTraffic{}).Error; err != nil {
  837. return false, err
  838. }
  839. }
  840. }
  841. if err := s.clientService.DetachInbound(tx, c.Id); err != nil {
  842. return false, err
  843. }
  844. if err := tx.Where("id = ?", c.Id).
  845. Delete(&model.Inbound{}).Error; err != nil {
  846. return false, err
  847. }
  848. delete(tagToCentral, c.Tag)
  849. structuralChange = true
  850. }
  851. for _, snapIb := range snap.Inbounds {
  852. if snapIb == nil {
  853. continue
  854. }
  855. c, ok := tagToCentral[snapIb.Tag]
  856. if !ok {
  857. continue
  858. }
  859. snapEmails := make(map[string]struct{}, len(snapIb.ClientStats))
  860. // Parsed once per inbound on the first renewal candidate, not per client.
  861. var snapExpiries map[string]int64
  862. for _, cs := range snapIb.ClientStats {
  863. snapEmails[cs.Email] = struct{}{}
  864. // Node-wide total, not this inbound's possibly-stale copy (#5274).
  865. canon := nodeEmailTotals[cs.Email]
  866. base, seen := nodeBaselines[cs.Email]
  867. var deltaUp, deltaDown int64
  868. if seen {
  869. if deltaUp = canon.Up - base.Up; deltaUp < 0 {
  870. deltaUp = 0
  871. }
  872. if deltaDown = canon.Down - base.Down; deltaDown < 0 {
  873. deltaDown = 0
  874. }
  875. }
  876. if _, rowExists := existingEmails[cs.Email]; !rowExists {
  877. if dirty {
  878. continue
  879. }
  880. _, isNewInbound := newInboundIDs[c.Id]
  881. // On a known inbound a missing row plus a live tombstone means the
  882. // master just deleted this client and the snapshot predates the
  883. // deletion push — recreating the row (at zero) would resurrect the
  884. // client. A freshly adopted inbound still gets its row (seeded at
  885. // zero) so adoption semantics stay intact.
  886. if !isNewInbound && isClientEmailTombstoned(cs.Email) {
  887. continue
  888. }
  889. var seedUp, seedDown int64
  890. if isNewInbound && !isClientEmailTombstoned(cs.Email) {
  891. seedUp, seedDown = canon.Up, canon.Down
  892. }
  893. row := &xray.ClientTraffic{
  894. InboundId: c.Id,
  895. Email: cs.Email,
  896. Enable: cs.Enable,
  897. Total: cs.Total,
  898. ExpiryTime: cs.ExpiryTime,
  899. Reset: cs.Reset,
  900. ResetDay: cs.ResetDay,
  901. Up: seedUp,
  902. Down: seedDown,
  903. LastOnline: cs.LastOnline,
  904. }
  905. if err := tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "email"}}, DoNothing: true}).
  906. Create(row).Error; err != nil {
  907. return false, err
  908. }
  909. centralCS[csKey{c.Id, cs.Email}] = row
  910. centralCSByEmail[cs.Email] = row
  911. existingEmails[cs.Email] = struct{}{}
  912. structuralChange = true
  913. if err := s.upsertNodeBaseline(tx, nodeID, cs.Email, canon.Up, canon.Down); err != nil {
  914. return false, err
  915. }
  916. nodeBaselines[cs.Email] = nodeTrafficCounter{Up: canon.Up, Down: canon.Down}
  917. continue
  918. }
  919. existing := centralCSByEmail[cs.Email]
  920. if existing != nil {
  921. expiryChanged := !lifecycleFrozen && existing.ExpiryTime != mergeActivationExpiry(existing.ExpiryTime, cs.ExpiryTime)
  922. // Only a real latch to disabled is structural; one-way merge never
  923. // re-enables from the node.
  924. enableChanged := !lifecycleFrozen && existing.Enable && !cs.Enable &&
  925. !nodeDisableIsStale(existing, cs, now, deltaUp, deltaDown)
  926. metaChanged := !lifecycleFrozen && (existing.Total != cs.Total || existing.Reset != cs.Reset)
  927. if enableChanged || metaChanged || expiryChanged {
  928. structuralChange = true
  929. }
  930. }
  931. renewed := !lifecycleFrozen && seen && existing != nil && nodeClientRenewed(existing, cs, canon, base)
  932. if renewed {
  933. // Reject when the node's own settings still carry the old absolute:
  934. // lagging ClientStats after a master shorten mimic a renew (#6228).
  935. if snapExpiries == nil {
  936. snapExpiries = settingsClientAbsoluteExpiries(snapIb.Settings)
  937. }
  938. if se, ok := snapExpiries[cs.Email]; ok && se <= existing.ExpiryTime {
  939. renewed = false
  940. }
  941. }
  942. if renewed {
  943. // A renewal starts a fresh quota window: adopt the node's counters
  944. // and enable state, drop stale pushes (mirrors autoRenewClients).
  945. if err := tx.Exec(
  946. fmt.Sprintf(
  947. `UPDATE client_traffics
  948. SET up = ?, down = ?, enable = ?, total = ?,
  949. expiry_time = ?, reset = ?, reset_day = ?, reset_count = ?, last_online = %s
  950. WHERE email = ?`,
  951. database.GreatestExpr("last_online", "?"),
  952. ),
  953. canon.Up, canon.Down, cs.Enable, cs.Total,
  954. cs.ExpiryTime, cs.Reset, cs.ResetDay, cs.ResetCount,
  955. cs.LastOnline, cs.Email,
  956. ).Error; err != nil {
  957. return false, err
  958. }
  959. if err := clearGlobalTraffic(tx, cs.Email); err != nil {
  960. return false, err
  961. }
  962. existing.Up = canon.Up
  963. existing.Down = canon.Down
  964. existing.Enable = cs.Enable
  965. existing.Total = cs.Total
  966. existing.ExpiryTime = cs.ExpiryTime
  967. existing.Reset = cs.Reset
  968. existing.ResetCount = cs.ResetCount
  969. structuralChange = true
  970. } else if lifecycleFrozen {
  971. // Push pending or just landed: only counters may move, the master
  972. // keeps expiry/enable/total/reset.
  973. if err := tx.Exec(
  974. fmt.Sprintf(
  975. `UPDATE client_traffics
  976. SET up = %s, down = %s, last_online = %s
  977. WHERE email = ?`,
  978. database.ClampedAddExpr("up"),
  979. database.ClampedAddExpr("down"),
  980. database.GreatestExpr("last_online", "?"),
  981. ),
  982. deltaUp, deltaDown, cs.LastOnline, cs.Email,
  983. ).Error; err != nil {
  984. return false, err
  985. }
  986. if existing != nil {
  987. existing.Up = clampTrafficCounter(existing.Up + deltaUp)
  988. existing.Down = clampTrafficCounter(existing.Down + deltaDown)
  989. }
  990. } else {
  991. enableExpr := database.ClientTrafficEnableMergeExpr()
  992. expiryExpr := database.ClientTrafficExpiryMergeExpr()
  993. if err := tx.Exec(
  994. fmt.Sprintf(
  995. `UPDATE client_traffics
  996. SET up = %s, down = %s, enable = %s, total = ?,
  997. expiry_time = %s,
  998. reset = ?, reset_day = ?, last_online = %s
  999. WHERE email = ?`,
  1000. database.ClampedAddExpr("up"),
  1001. database.ClampedAddExpr("down"),
  1002. enableExpr,
  1003. expiryExpr,
  1004. database.GreatestExpr("last_online", "?"),
  1005. ),
  1006. deltaUp, deltaDown,
  1007. cs.Enable, cs.ExpiryTime, cs.Total, now, deltaUp, deltaDown,
  1008. cs.Total,
  1009. cs.ExpiryTime, cs.Reset, cs.ResetDay,
  1010. cs.LastOnline, cs.Email,
  1011. ).Error; err != nil {
  1012. return false, err
  1013. }
  1014. if existing != nil {
  1015. priorExpiry := existing.ExpiryTime
  1016. if !cs.Enable && !nodeDisableIsStale(existing, cs, now, deltaUp, deltaDown) {
  1017. existing.Enable = false
  1018. }
  1019. existing.ExpiryTime = mergeActivationExpiry(priorExpiry, cs.ExpiryTime)
  1020. existing.Up = clampTrafficCounter(existing.Up + deltaUp)
  1021. existing.Down = clampTrafficCounter(existing.Down + deltaDown)
  1022. existing.Total = cs.Total
  1023. existing.Reset = cs.Reset
  1024. }
  1025. }
  1026. // A dip plus a lagging longer expiry mimics nodeClientRenewed and would
  1027. // undo a master shorten once the freeze lifts (#6228).
  1028. if lifecycleFrozen && seen && (canon.Up < base.Up || canon.Down < base.Down) {
  1029. continue
  1030. }
  1031. if err := s.upsertNodeBaseline(tx, nodeID, cs.Email, canon.Up, canon.Down); err != nil {
  1032. return false, err
  1033. }
  1034. nodeBaselines[cs.Email] = nodeTrafficCounter{Up: canon.Up, Down: canon.Down}
  1035. }
  1036. for k, existing := range centralCS {
  1037. if dirty {
  1038. continue
  1039. }
  1040. if k.inboundID != c.Id {
  1041. continue
  1042. }
  1043. if _, kept := snapEmails[k.email]; kept {
  1044. continue
  1045. }
  1046. // Gone from this inbound's stats but still reported by the node under
  1047. // a sibling inbound: both the shared accumulator row and the (node,
  1048. // email) baseline must survive, or the sibling's next delta would
  1049. // compute against nothing and freeze the counter (#5202).
  1050. if _, still := snapEmailsAll[k.email]; still {
  1051. continue
  1052. }
  1053. if err := tx.Where("node_id = ? AND email = ?", nodeID, existing.Email).
  1054. Delete(&model.NodeClientTraffic{}).Error; err != nil {
  1055. return false, err
  1056. }
  1057. // Same shared-accumulator rule as the inbound-removal sweep above:
  1058. // keep the row while another inbound still references the email.
  1059. stillUsed, uErr := s.emailUsedByOtherInbounds(existing.Email, c.Id)
  1060. if uErr != nil {
  1061. return false, uErr
  1062. }
  1063. // Usage, quota and expiry live on this row, so a client the orphan
  1064. // sweep will mark keeps it until the reaper confirms the removal.
  1065. if !stillUsed && !clientRecordExists(tx, existing.Email) {
  1066. if err := tx.Where("inbound_id = ? AND email = ?", c.Id, existing.Email).
  1067. Delete(&xray.ClientTraffic{}).Error; err != nil {
  1068. return false, err
  1069. }
  1070. }
  1071. structuralChange = true
  1072. }
  1073. }
  1074. type oldSet struct {
  1075. inboundID int
  1076. emails map[string]struct{}
  1077. }
  1078. var perInboundOld []oldSet
  1079. syncFailedInbounds := map[int]struct{}{}
  1080. for _, p := range pendingAdopts {
  1081. lifted, liftChanged := liftClientLifecycleInSettings(p.wireSettings, centralCSByEmail)
  1082. adoptedSettings := p.wireSettings
  1083. if liftChanged {
  1084. adoptedSettings = lifted
  1085. lifecycleLifted = true
  1086. }
  1087. if p.central.Settings != adoptedSettings {
  1088. if err := tx.Model(model.Inbound{}).
  1089. Where("id = ?", p.central.Id).
  1090. Update("settings", adoptedSettings).Error; err != nil {
  1091. return false, err
  1092. }
  1093. structuralChange = true
  1094. }
  1095. // The fingerprint stamps the un-lifted wire blob on purpose: a lift must
  1096. // leave reconcile a mismatch to re-push against.
  1097. if liftChanged || adoptedWireChanged(p.central, p.snapIb, p.wireSettings) {
  1098. adoptedInbounds = append(adoptedInbounds, adoptedWireInbound(p.central, p.snapIb, p.wireSettings))
  1099. }
  1100. }
  1101. for _, snapIb := range snap.Inbounds {
  1102. if snapIb == nil {
  1103. continue
  1104. }
  1105. c, ok := tagToCentral[snapIb.Tag]
  1106. if !ok {
  1107. continue
  1108. }
  1109. if dirty {
  1110. continue
  1111. }
  1112. var oldEmailsRows []string
  1113. if err := tx.Table("clients").
  1114. Joins("JOIN client_inbounds ON client_inbounds.client_id = clients.id").
  1115. Where("client_inbounds.inbound_id = ?", c.Id).
  1116. Pluck("email", &oldEmailsRows).Error; err == nil {
  1117. oldEmails := make(map[string]struct{}, len(oldEmailsRows))
  1118. for _, e := range oldEmailsRows {
  1119. if e != "" {
  1120. oldEmails[e] = struct{}{}
  1121. }
  1122. }
  1123. perInboundOld = append(perInboundOld, oldSet{inboundID: c.Id, emails: oldEmails})
  1124. }
  1125. clients, gcErr := s.GetClients(snapIb)
  1126. if gcErr != nil {
  1127. logger.Warningf("setRemoteTraffic: parse clients for tag %q failed: %v", snapIb.Tag, gcErr)
  1128. continue
  1129. }
  1130. csByEmail := make(map[string]xray.ClientTraffic, len(snapIb.ClientStats))
  1131. for _, cs := range snapIb.ClientStats {
  1132. csByEmail[cs.Email] = cs
  1133. }
  1134. filtered := clients[:0]
  1135. for i := range clients {
  1136. if isClientEmailTombstoned(clients[i].Email) {
  1137. continue
  1138. }
  1139. existing := centralCSByEmail[clients[i].Email]
  1140. var csPtr *xray.ClientTraffic
  1141. if cs, hit := csByEmail[clients[i].Email]; hit {
  1142. csCopy := cs
  1143. csPtr = &csCopy
  1144. }
  1145. applyMasterClientLifecycle(&clients[i], existing, csPtr)
  1146. filtered = append(filtered, clients[i])
  1147. }
  1148. localEmails := make([]string, 0, len(filtered))
  1149. for i := range filtered {
  1150. if filtered[i].Email != "" {
  1151. localEmails = append(localEmails, filtered[i].Email)
  1152. }
  1153. }
  1154. if len(localEmails) > 0 {
  1155. var localMeta []struct {
  1156. Email string
  1157. Comment string `gorm:"column:comment"`
  1158. }
  1159. if err := tx.Table("clients").
  1160. Select("email, comment").
  1161. Where("email IN ?", localEmails).
  1162. Find(&localMeta).Error; err == nil {
  1163. commentByEmail := make(map[string]string, len(localMeta))
  1164. for _, m := range localMeta {
  1165. commentByEmail[m.Email] = m.Comment
  1166. }
  1167. for i := range filtered {
  1168. if cmt, ok := commentByEmail[filtered[i].Email]; ok {
  1169. filtered[i].Comment = cmt
  1170. }
  1171. }
  1172. }
  1173. }
  1174. if len(localEmails) > 0 {
  1175. foreign, err := clientEmailsOwnedElsewhere(tx, nodeID, localEmails)
  1176. if err != nil {
  1177. return false, err
  1178. }
  1179. if len(foreign) > 0 {
  1180. kept := filtered[:0]
  1181. for i := range filtered {
  1182. if _, claimed := foreign[filtered[i].Email]; !claimed {
  1183. kept = append(kept, filtered[i])
  1184. continue
  1185. }
  1186. key := fmt.Sprintf("%d:%s", nodeID, filtered[i].Email)
  1187. if _, seen := reportedForeignClientClaim.LoadOrStore(key, struct{}{}); !seen {
  1188. logger.Warningf(
  1189. "setRemoteTraffic: node %d reported client %q, which is attached only to inbounds of another node — not adopting (rename one side to remove the duplicate email)",
  1190. nodeID, filtered[i].Email,
  1191. )
  1192. }
  1193. }
  1194. filtered = kept
  1195. }
  1196. }
  1197. if err := s.clientService.SyncInbound(tx, c.Id, filtered); err != nil {
  1198. logger.Warningf("setRemoteTraffic: sync clients for tag %q failed: %v", snapIb.Tag, err)
  1199. syncFailedInbounds[c.Id] = struct{}{}
  1200. }
  1201. }
  1202. for _, old := range perInboundOld {
  1203. // The sweep's premise is that links were just rebuilt from the snapshot,
  1204. // which is exactly what a failed SyncInbound violates.
  1205. if _, failed := syncFailedInbounds[old.inboundID]; failed {
  1206. continue
  1207. }
  1208. var stillAttached []string
  1209. if err := tx.Table("clients").
  1210. Joins("JOIN client_inbounds ON client_inbounds.client_id = clients.id").
  1211. Where("client_inbounds.inbound_id = ?", old.inboundID).
  1212. Pluck("email", &stillAttached).Error; err != nil {
  1213. continue
  1214. }
  1215. stillSet := make(map[string]struct{}, len(stillAttached))
  1216. for _, e := range stillAttached {
  1217. stillSet[e] = struct{}{}
  1218. }
  1219. for email := range old.emails {
  1220. if _, kept := stillSet[email]; kept {
  1221. continue
  1222. }
  1223. var attachmentCount int64
  1224. if err := tx.Table("client_inbounds").
  1225. Joins("JOIN clients ON clients.id = client_inbounds.client_id").
  1226. Where("clients.email = ?", email).
  1227. Count(&attachmentCount).Error; err != nil {
  1228. continue
  1229. }
  1230. if attachmentCount > 0 {
  1231. continue
  1232. }
  1233. // "Ended the merge unattached" is true for a real remote deletion and
  1234. // equally true for a bad merge, so record a strike instead of deleting.
  1235. if err := markSyncOrphan(tx, email, now); err != nil {
  1236. logger.Warningf("setRemoteTraffic: mark orphan %q failed: %v", email, err)
  1237. continue
  1238. }
  1239. structuralChange = true
  1240. }
  1241. }
  1242. if err := clearSyncOrphanMarks(tx); err != nil {
  1243. logger.Warning("setRemoteTraffic: clear orphan marks failed:", err)
  1244. }
  1245. if err := liftActivatedClientRecordExpiries(tx); err != nil {
  1246. logger.Warning("setRemoteTraffic: lift activated expiries failed:", err)
  1247. }
  1248. if err := tx.Commit().Error; err != nil {
  1249. return false, err
  1250. }
  1251. committed = true
  1252. if lifecycleLifted && !dirty {
  1253. var already model.Node
  1254. if err := database.GetDB().Select("config_dirty").Where("id = ?", nodeID).First(&already).Error; err == nil && already.ConfigDirty {
  1255. logger.Debugf("setRemoteTraffic: node %d lifecycle lift; already dirty", nodeID)
  1256. } else {
  1257. logger.Infof("setRemoteTraffic: node %d lifecycle lift; marking dirty for re-push", nodeID)
  1258. if err := (&NodeService{}).MarkNodeDirty(nodeID); err != nil {
  1259. logger.Warningf("setRemoteTraffic: mark node %d dirty after lifecycle lift failed: %v", nodeID, err)
  1260. }
  1261. }
  1262. }
  1263. if len(adoptedInbounds) > 0 {
  1264. if mgr := runtime.GetManager(); mgr != nil {
  1265. if rt, rtErr := mgr.RuntimeFor(&nodeID); rtErr == nil {
  1266. if rem, ok := rt.(*runtime.Remote); ok {
  1267. for _, ib := range adoptedInbounds {
  1268. rem.RecordAdoptedInbound(ib)
  1269. }
  1270. }
  1271. }
  1272. }
  1273. }
  1274. if process := currentXrayProcess(); process != nil {
  1275. tree := snap.OnlineTree
  1276. switch {
  1277. case len(tree) == 0 && len(snap.OnlineEmails) > 0:
  1278. // Old-build node (no GUID tree): key its flat online list under its
  1279. // own effective identity so attribution still works for that branch.
  1280. tree = map[string][]string{selfKey: snap.OnlineEmails}
  1281. case guidShared && len(tree) > 0:
  1282. // Newer cloned node: its own clients arrive keyed under the shared
  1283. // panelGuid. Remap just that entry to the node-unique key so the
  1284. // clones don't merge; descendant subtrees keep their distinct GUIDs.
  1285. if _, ok := tree[nodeRow.Guid]; ok {
  1286. tree = remapGuidTreeKey(tree, nodeRow.Guid, selfKey)
  1287. }
  1288. }
  1289. process.SetNodeOnlineTree(nodeID, tree)
  1290. activeTree := normalizeActiveInboundTreeTags(snap.ActiveInboundTree, tagToCentral)
  1291. if guidShared && len(activeTree) > 0 {
  1292. if _, ok := activeTree[nodeRow.Guid]; ok {
  1293. activeTree = remapGuidTreeKey(activeTree, nodeRow.Guid, selfKey)
  1294. }
  1295. }
  1296. if len(activeTree) > 0 {
  1297. activeTree = filterGuidTreeKeys(activeTree, activeInboundGuidKeys(snap.Inbounds, tagToCentral, originGuidFor))
  1298. }
  1299. process.SetNodeActiveInboundTree(nodeID, activeTree)
  1300. }
  1301. return structuralChange, nil
  1302. }
  1303. func (s *InboundService) restartRemoteNodesOnDisable(nodeIDs []int) {
  1304. restartOnDisable, err := (&SettingService{}).GetRestartXrayOnClientDisable()
  1305. if err != nil {
  1306. logger.Warning("disableInvalidClients: get RestartXrayOnClientDisable failed:", err)
  1307. return
  1308. }
  1309. if !restartOnDisable {
  1310. return
  1311. }
  1312. // Best-effort and never replayed: a hanging node must not hold the traffic poll.
  1313. common.GoRecover("restart-nodes-on-client-disable", func() {
  1314. for _, nodeID := range nodeIDs {
  1315. nodeIDCopy := nodeID
  1316. rt, rtErr := runtime.GetManager().RuntimeFor(&nodeIDCopy)
  1317. if rtErr != nil {
  1318. logger.Warning("disableInvalidClients: get runtime for node", nodeID, "failed:", rtErr)
  1319. continue
  1320. }
  1321. if rtErr = rt.RestartXray(context.Background()); rtErr != nil {
  1322. logger.Warning("disableInvalidClients: restart xray on node", nodeID, "failed:", rtErr)
  1323. }
  1324. }
  1325. })
  1326. }
  1327. func (s *InboundService) GetOnlineClients() []string {
  1328. process := currentXrayProcess()
  1329. if process == nil {
  1330. return []string{}
  1331. }
  1332. return process.GetOnlineClients()
  1333. }
  1334. // GetOnlineClientsByGuid returns online emails keyed by the panelGuid of the
  1335. // node that physically hosts each set: this panel's own clients under its own
  1336. // GUID, plus every node in the tree under its GUID (#4983). Replaces the old
  1337. // node-id keying so a client three hops down is attributed to its real node,
  1338. // not the intermediate one it was synced through.
  1339. func (s *InboundService) GetOnlineClientsByGuid() map[string][]string {
  1340. process := currentXrayProcess()
  1341. if process == nil {
  1342. return map[string][]string{}
  1343. }
  1344. out := process.GetMergedNodeTrees()
  1345. if local := process.GetLocalOnlineClients(); len(local) > 0 {
  1346. if guid := s.panelGuid(); guid != "" {
  1347. out[guid] = mergeEmails(out[guid], local)
  1348. }
  1349. }
  1350. return out
  1351. }
  1352. // GetActiveInboundsByGuid returns the inbound tags that carried traffic within
  1353. // the grace window, keyed by the panelGuid of the node that physically hosts
  1354. // each inbound. A GUID missing from the map means "don't gate" for that node's
  1355. // inbounds (old-build node or no active-inbound signal).
  1356. func (s *InboundService) GetActiveInboundsByGuid() map[string][]string {
  1357. process := currentXrayProcess()
  1358. if process == nil {
  1359. return map[string][]string{}
  1360. }
  1361. out := process.GetMergedActiveInboundTrees()
  1362. active := process.GetLocalActiveInbounds()
  1363. if len(active) == 0 {
  1364. return out
  1365. }
  1366. guid := s.panelGuid()
  1367. if guid == "" {
  1368. return out
  1369. }
  1370. out[guid] = mergeEmails(out[guid], active)
  1371. return out
  1372. }
  1373. func (s *InboundService) SetNodeOnlineTree(nodeID int, tree map[string][]string) {
  1374. if process := currentXrayProcess(); process != nil {
  1375. process.SetNodeOnlineTree(nodeID, tree)
  1376. }
  1377. }
  1378. func (s *InboundService) ClearNodeOnlineClients(nodeID int) {
  1379. if process := currentXrayProcess(); process != nil {
  1380. process.ClearNodeOnlineClients(nodeID)
  1381. }
  1382. }
  1383. // panelGuid returns this panel's stable self-identifier, used to key the local
  1384. // panel's own clients in the per-node online maps (#4983).
  1385. func (s *InboundService) panelGuid() string {
  1386. guid, _ := (&SettingService{}).GetPanelGuid()
  1387. return guid
  1388. }
  1389. // synthNodeGuid is the stable per-node fallback identity for a directly-attached
  1390. // node whose panel hasn't reported a panelGuid yet (old build). Node ids are
  1391. // master-local, so this only composes for direct nodes — exactly the pre-#4983
  1392. // flat-topology case where an old-build node appears.
  1393. func synthNodeGuid(nodeID int) string {
  1394. return fmt.Sprintf("node:%d", nodeID)
  1395. }
  1396. // mergeEmails returns the deduped union of two email slices.
  1397. func mergeEmails(a, b []string) []string {
  1398. if len(a) == 0 {
  1399. return b
  1400. }
  1401. seen := make(map[string]struct{}, len(a)+len(b))
  1402. out := make([]string, 0, len(a)+len(b))
  1403. for _, e := range a {
  1404. if _, ok := seen[e]; !ok {
  1405. seen[e] = struct{}{}
  1406. out = append(out, e)
  1407. }
  1408. }
  1409. for _, e := range b {
  1410. if _, ok := seen[e]; !ok {
  1411. seen[e] = struct{}{}
  1412. out = append(out, e)
  1413. }
  1414. }
  1415. return out
  1416. }
  1417. func remapGuidTreeKey(tree map[string][]string, from, to string) map[string][]string {
  1418. if from == "" || to == "" || from == to {
  1419. return tree
  1420. }
  1421. remapped := make(map[string][]string, len(tree))
  1422. for guid, values := range tree {
  1423. if guid == from {
  1424. guid = to
  1425. }
  1426. remapped[guid] = mergeEmails(remapped[guid], values)
  1427. }
  1428. return remapped
  1429. }
  1430. func normalizeActiveInboundTreeTags(tree map[string][]string, tagToCentral map[string]*model.Inbound) map[string][]string {
  1431. if len(tree) == 0 {
  1432. return nil
  1433. }
  1434. out := make(map[string][]string, len(tree))
  1435. for guid, tags := range tree {
  1436. if guid == "" || len(tags) == 0 {
  1437. continue
  1438. }
  1439. seen := make(map[string]struct{}, len(tags))
  1440. for _, tag := range tags {
  1441. if tag == "" {
  1442. continue
  1443. }
  1444. if central, ok := tagToCentral[tag]; ok && central != nil && central.Tag != "" {
  1445. tag = central.Tag
  1446. }
  1447. if _, dup := seen[tag]; dup {
  1448. continue
  1449. }
  1450. seen[tag] = struct{}{}
  1451. out[guid] = append(out[guid], tag)
  1452. }
  1453. }
  1454. if len(out) == 0 {
  1455. return nil
  1456. }
  1457. return out
  1458. }
  1459. func activeInboundGuidKeys(inbounds []*model.Inbound, tagToCentral map[string]*model.Inbound, originGuidFor func(*model.Inbound) string) map[string]struct{} {
  1460. allowed := make(map[string]struct{})
  1461. for _, ib := range inbounds {
  1462. if ib == nil {
  1463. continue
  1464. }
  1465. if _, ok := tagToCentral[ib.Tag]; !ok {
  1466. continue
  1467. }
  1468. if guid := originGuidFor(ib); guid != "" {
  1469. allowed[guid] = struct{}{}
  1470. }
  1471. }
  1472. return allowed
  1473. }
  1474. func filterGuidTreeKeys(tree map[string][]string, allowed map[string]struct{}) map[string][]string {
  1475. if len(tree) == 0 || len(allowed) == 0 {
  1476. return nil
  1477. }
  1478. out := make(map[string][]string, len(tree))
  1479. for guid, values := range tree {
  1480. if _, ok := allowed[guid]; !ok {
  1481. continue
  1482. }
  1483. if len(values) > 0 {
  1484. out[guid] = values
  1485. }
  1486. }
  1487. if len(out) == 0 {
  1488. return nil
  1489. }
  1490. return out
  1491. }
  1492. func (s *InboundService) GetClientsLastOnline() (map[string]int64, error) {
  1493. db := database.GetDB()
  1494. var rows []xray.ClientTraffic
  1495. err := db.Model(&xray.ClientTraffic{}).Select("email, last_online").Find(&rows).Error
  1496. if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
  1497. return nil, err
  1498. }
  1499. result := make(map[string]int64, len(rows))
  1500. for _, r := range rows {
  1501. result[r.Email] = r.LastOnline
  1502. }
  1503. return result, nil
  1504. }
  1505. // RefreshLocalOnlineClients folds the emails and inbound tags active on this
  1506. // panel's own xray this poll into the local online/active sets, applying the
  1507. // grace window and pruning stale entries. Pass nil to only prune. See
  1508. // xray.Process for why the local sets are kept separate from the shared
  1509. // last_online column.
  1510. func (s *InboundService) RefreshLocalOnlineClients(activeEmails, activeInboundTags []string) {
  1511. if process := currentXrayProcess(); process != nil {
  1512. process.RefreshLocalOnline(activeEmails, activeInboundTags, time.Now().UnixMilli(), onlineGracePeriodMs)
  1513. }
  1514. }
  1515. func (s *InboundService) FilterAndSortClientEmails(emails []string) ([]string, []string, error) {
  1516. db := database.GetDB()
  1517. // Step 1: Get ClientTraffic records for emails in the input list.
  1518. // Chunked to stay under SQLite's bind-variable limit on huge inputs.
  1519. uniqEmails := uniqueNonEmptyStrings(emails)
  1520. clients := make([]xray.ClientTraffic, 0, len(uniqEmails))
  1521. for _, batch := range chunkStrings(uniqEmails, sqliteMaxVars) {
  1522. var page []xray.ClientTraffic
  1523. if err := db.Where("email IN ?", batch).Find(&page).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
  1524. return nil, nil, err
  1525. }
  1526. clients = append(clients, page...)
  1527. }
  1528. // Step 2: Sort clients by (Up + Down) descending
  1529. sort.Slice(clients, func(i, j int) bool {
  1530. return (clients[i].Up + clients[i].Down) > (clients[j].Up + clients[j].Down)
  1531. })
  1532. // Step 3: Extract sorted valid emails and track found ones
  1533. validEmails := make([]string, 0, len(clients))
  1534. found := make(map[string]bool)
  1535. for _, client := range clients {
  1536. validEmails = append(validEmails, client.Email)
  1537. found[client.Email] = true
  1538. }
  1539. // Step 4: Identify emails that were not found in the database
  1540. extraEmails := make([]string, 0)
  1541. for _, email := range emails {
  1542. if !found[email] {
  1543. extraEmails = append(extraEmails, email)
  1544. }
  1545. }
  1546. return validEmails, extraEmails, nil
  1547. }