gfcp.go 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533
  1. // Package gfcp - A Fast and Reliable ARQ Protocol
  2. //
  3. // Copyright © 2015 Daniel Fu <daniel820313@gmail.com>.
  4. // Copyright © 2019 Loki 'l0k18' Verloren <stalker.loki@protonmail.ch>.
  5. // Copyright © 2021 Gridfinity, LLC. <admin@gridfinity.com>.
  6. //
  7. // All rights reserved.
  8. //
  9. // All use of this code is governed by the MIT license.
  10. // The complete license is available in the LICENSE file.
  11. package gfcp // import "go.gridfinity.dev/gfcp"
  12. import (
  13. "encoding/binary"
  14. "math"
  15. "runtime/debug"
  16. "sync/atomic"
  17. gfcpLegal "go4.org/legal"
  18. )
  19. // Gfcp protocol constants
  20. const (
  21. GfcpRtoNdl = 20 // GfcpRtoNdl: NoDelay min RTO
  22. GfcpRtoMin = 120 // GfcpRtoMin: Regular min RTO
  23. GfcpRtoDef = 340
  24. GfcpRtoMax = 60000
  25. GfcpCmdPush = 81 // GfcpCmdPush: Push data
  26. GfcpCmdAck = 82 // GfcpCmdAck: Ack
  27. GfcpCmdWask = 83 // GfcpCmdWask: Get Window Size
  28. GfcpCmdWins = 84 // GfcpCmdWins: Set window Size
  29. GfcpAskSend = 1 // GfcpAskSend: Need to send GfcpCmdWask
  30. GfcpAskTell = 2 // GfcpAskTell: Need to send GfcpCmdWins
  31. GfcpWndSnd = 32
  32. GfcpWndRcv = 32
  33. GfcpMtuDef = 1480
  34. GfcpAckFast = 3
  35. GfcpInterval = 100
  36. GfcpOverhead = 24
  37. GfcpDeadLink = 20
  38. GfcpThreshInit = 2
  39. GfcpThreshMin = 2
  40. GfcpProbeInit = 7000 // 7s initial probe window
  41. GfcpProbeLimit = 102000 // 120s hard probe timeout
  42. )
  43. type outputCallback func(
  44. buf []byte,
  45. size int,
  46. )
  47. func gfcpEncode8u(
  48. p []byte,
  49. c byte,
  50. ) []byte {
  51. p[0] = c
  52. return p[1:]
  53. }
  54. func gfcpDecode8u(
  55. p []byte,
  56. c *byte,
  57. ) []byte {
  58. *c = p[0]
  59. return p[1:]
  60. }
  61. func gfcpEncode16u(
  62. p []byte,
  63. w uint16,
  64. ) []byte {
  65. binary.LittleEndian.PutUint16(
  66. p,
  67. w,
  68. )
  69. return p[2:]
  70. }
  71. func gfcpDecode16u(
  72. p []byte,
  73. w *uint16,
  74. ) []byte {
  75. *w = binary.LittleEndian.Uint16(
  76. p,
  77. )
  78. return p[2:]
  79. }
  80. func gfcpEncode32u(
  81. p []byte,
  82. l uint32,
  83. ) []byte {
  84. binary.LittleEndian.PutUint32(
  85. p,
  86. l,
  87. )
  88. return p[4:]
  89. }
  90. func gfcpDecode32u(
  91. p []byte,
  92. l *uint32,
  93. ) []byte {
  94. *l = binary.LittleEndian.Uint32(
  95. p,
  96. )
  97. return p[4:]
  98. }
  99. func _imin(
  100. a,
  101. b uint32,
  102. ) uint32 {
  103. if a <= b {
  104. return a
  105. }
  106. return b
  107. }
  108. func _imax(
  109. a,
  110. b uint32,
  111. ) uint32 {
  112. if a >= b {
  113. return a
  114. }
  115. return b
  116. }
  117. func _ibound(
  118. lower,
  119. middle,
  120. upper uint32,
  121. ) uint32 {
  122. return _imin(
  123. _imax(
  124. lower,
  125. middle,
  126. ),
  127. upper,
  128. )
  129. }
  130. func _itimediff(
  131. later,
  132. earlier uint32,
  133. ) int32 {
  134. return (int32)(later - earlier)
  135. }
  136. // Segment structure
  137. type Segment struct {
  138. conv uint32
  139. cmd uint8
  140. frg uint8
  141. wnd uint16
  142. ts uint32
  143. sn uint32
  144. una uint32
  145. rto uint32
  146. Kxmit uint32
  147. GFcpResendTs uint32
  148. fastack uint32
  149. acked uint32
  150. data []byte
  151. }
  152. func (
  153. GFcpSeg *Segment,
  154. ) encode(
  155. ptr []byte,
  156. ) []byte {
  157. ptr = gfcpEncode32u(
  158. ptr,
  159. GFcpSeg.conv,
  160. )
  161. ptr = gfcpEncode8u(
  162. ptr,
  163. GFcpSeg.cmd,
  164. )
  165. ptr = gfcpEncode8u(
  166. ptr,
  167. GFcpSeg.frg,
  168. )
  169. ptr = gfcpEncode16u(
  170. ptr,
  171. GFcpSeg.wnd,
  172. )
  173. ptr = gfcpEncode32u(
  174. ptr,
  175. GFcpSeg.ts,
  176. )
  177. ptr = gfcpEncode32u(
  178. ptr,
  179. GFcpSeg.sn,
  180. )
  181. ptr = gfcpEncode32u(
  182. ptr,
  183. GFcpSeg.una,
  184. )
  185. ptr = gfcpEncode32u(
  186. ptr, uint32(len(
  187. GFcpSeg.data,
  188. )))
  189. atomic.AddUint64(
  190. &DefaultSnsi.GFcpOutputSegments,
  191. 1,
  192. )
  193. return ptr
  194. }
  195. // GFCP primary structure
  196. type GFCP struct {
  197. conv, mtu, mss, state uint32
  198. sndUna, sndNxt, rcvNxt uint32
  199. ssthresh uint32
  200. rxRttVar, rxSrtt int32
  201. rxRto, rxMinRto uint32
  202. sndWnd, rcvWnd, rmtWnd, cwnd, probe uint32
  203. interval, tsFlush uint32
  204. nodelay, updated uint32
  205. tsProbe, probeWait uint32
  206. deadLink, incr uint32
  207. fastresend int32
  208. nocwnd, stream int32
  209. sndQueue []Segment
  210. rcvQueue []Segment
  211. SndBuf []Segment
  212. rcvBuf []Segment
  213. acklist []ackItem
  214. buffer []byte
  215. reserved int
  216. output outputCallback
  217. }
  218. type ackItem struct {
  219. sn uint32
  220. ts uint32
  221. }
  222. // NewGFCP creates a new GFcp control object.
  223. func NewGFCP(
  224. conv uint32,
  225. output outputCallback,
  226. ) *GFCP {
  227. GFcp := new(
  228. GFCP,
  229. )
  230. GFcp.conv = conv
  231. GFcp.sndWnd = GfcpWndSnd
  232. GFcp.rcvWnd = GfcpWndRcv
  233. GFcp.rmtWnd = GfcpWndRcv
  234. GFcp.mtu = GfcpMtuDef
  235. GFcp.mss = GFcp.mtu - GfcpOverhead
  236. GFcp.buffer = make(
  237. []byte,
  238. GFcp.mtu,
  239. )
  240. GFcp.rxRto = GfcpRtoDef
  241. GFcp.rxMinRto = GfcpRtoMin
  242. GFcp.interval = GfcpInterval
  243. GFcp.tsFlush = GfcpInterval
  244. GFcp.ssthresh = GfcpThreshInit
  245. GFcp.deadLink = GfcpDeadLink
  246. GFcp.output = output
  247. return GFcp
  248. }
  249. func (
  250. GFcp *GFCP,
  251. ) newSegment(
  252. size int,
  253. ) (
  254. GFcpSeg Segment,
  255. ) {
  256. GFcpSeg.data = KxmitBuf.Get().([]byte)[:size]
  257. return
  258. }
  259. func (
  260. GFcp *GFCP,
  261. ) delSegment(
  262. GFcpSeg *Segment,
  263. ) {
  264. if GFcpSeg.data != nil {
  265. KxmitBuf.Put(
  266. // TODO(jhj): Switch to pointer to avoid allocation
  267. GFcpSeg.data,
  268. )
  269. GFcpSeg.data = nil
  270. }
  271. }
  272. // ReserveBytes keeps 'n' bytes from the beginning of buffering.
  273. // Output callbacks use this to return 'false' if 'n' >= 'mss'.
  274. func (
  275. GFcp *GFCP,
  276. ) ReserveBytes(
  277. n int,
  278. ) bool {
  279. if n >= int(
  280. GFcp.mtu-GfcpOverhead,
  281. ) || n < 0 {
  282. return false
  283. }
  284. GFcp.reserved = n
  285. GFcp.mss = GFcp.mtu - GfcpOverhead - uint32(
  286. n,
  287. )
  288. return true
  289. }
  290. // PeekSize checks the size of next message in the receive queue.
  291. func (
  292. GFcp *GFCP,
  293. ) PeekSize() (
  294. length int,
  295. ) {
  296. if len(
  297. GFcp.rcvQueue,
  298. ) == 0 {
  299. return -1
  300. }
  301. GFcpSeg := &GFcp.rcvQueue[0]
  302. if GFcpSeg.frg == 0 {
  303. return len(
  304. GFcpSeg.data,
  305. )
  306. }
  307. if len(
  308. GFcp.rcvQueue,
  309. ) < int(
  310. GFcpSeg.frg+1,
  311. ) {
  312. return -1
  313. }
  314. for k := range GFcp.rcvQueue {
  315. GFcpSeg := &GFcp.rcvQueue[k]
  316. length += len(
  317. GFcpSeg.data,
  318. )
  319. if GFcpSeg.frg == 0 {
  320. break
  321. }
  322. }
  323. return
  324. }
  325. // Recv is upper level recviver; returns size or EAGAIN on error.
  326. func (
  327. GFcp *GFCP,
  328. ) Recv(
  329. buffer []byte,
  330. ) (
  331. n int,
  332. ) {
  333. if len(
  334. GFcp.rcvQueue,
  335. ) == 0 {
  336. return -1
  337. }
  338. peeksize := GFcp.PeekSize()
  339. if peeksize < 0 {
  340. return -2
  341. }
  342. if peeksize > len(
  343. buffer,
  344. ) {
  345. return -3
  346. }
  347. var fastRecovery bool
  348. if len(
  349. GFcp.rcvQueue,
  350. ) >= int(
  351. GFcp.rcvWnd,
  352. ) {
  353. fastRecovery = true
  354. }
  355. count := 0
  356. for k := range GFcp.rcvQueue {
  357. GFcpSeg := &GFcp.rcvQueue[k]
  358. copy(
  359. buffer,
  360. GFcpSeg.data,
  361. )
  362. buffer = buffer[len(
  363. GFcpSeg.data,
  364. ):]
  365. n += len(
  366. GFcpSeg.data,
  367. )
  368. count++
  369. GFcp.delSegment(
  370. GFcpSeg,
  371. )
  372. if GFcpSeg.frg == 0 {
  373. break
  374. }
  375. }
  376. if count > 0 {
  377. GFcp.rcvQueue = GFcp.removeFront(
  378. GFcp.rcvQueue,
  379. count,
  380. )
  381. }
  382. count = 0
  383. for k := range GFcp.rcvBuf {
  384. GFcpSeg := &GFcp.rcvBuf[k]
  385. if GFcpSeg.sn == GFcp.rcvNxt && len(
  386. GFcp.rcvQueue,
  387. ) < int(
  388. GFcp.rcvWnd,
  389. ) {
  390. GFcp.rcvNxt++
  391. count++
  392. } else {
  393. break
  394. }
  395. }
  396. if count > 0 {
  397. GFcp.rcvQueue = append(
  398. GFcp.rcvQueue,
  399. GFcp.rcvBuf[:count]...,
  400. )
  401. GFcp.rcvBuf = GFcp.removeFront(
  402. GFcp.rcvBuf,
  403. count,
  404. )
  405. }
  406. if len(
  407. GFcp.rcvQueue,
  408. ) < int(
  409. GFcp.rcvWnd,
  410. ) && fastRecovery {
  411. GFcp.probe |= GfcpAskTell
  412. }
  413. return
  414. }
  415. // Send is upper level sender, returns <0 on error.
  416. func (
  417. GFcp *GFCP,
  418. ) Send(
  419. buffer []byte,
  420. ) int {
  421. var count int
  422. if len(
  423. buffer,
  424. ) == 0 {
  425. return -1
  426. }
  427. if GFcp.stream != 0 {
  428. n := len(
  429. GFcp.sndQueue,
  430. )
  431. if n > 0 {
  432. GFcpSeg := &GFcp.sndQueue[n-1]
  433. if len(
  434. GFcpSeg.data,
  435. ) < int(
  436. GFcp.mss,
  437. ) {
  438. capacity := int(
  439. GFcp.mss,
  440. ) - len(
  441. GFcpSeg.data,
  442. )
  443. extend := capacity
  444. if len(
  445. buffer,
  446. ) < capacity {
  447. extend = len(
  448. buffer,
  449. )
  450. }
  451. oldlen := len(
  452. GFcpSeg.data,
  453. )
  454. GFcpSeg.data = GFcpSeg.data[:oldlen+extend]
  455. copy(
  456. GFcpSeg.data[oldlen:],
  457. buffer,
  458. )
  459. buffer = buffer[extend:]
  460. }
  461. }
  462. if len(
  463. buffer,
  464. ) == 0 {
  465. return 0
  466. }
  467. }
  468. if len(
  469. buffer,
  470. ) <= int(
  471. GFcp.mss,
  472. ) {
  473. count = 1
  474. } else {
  475. count = (len(
  476. buffer,
  477. ) + int(
  478. GFcp.mss,
  479. ) - 1) / int(
  480. GFcp.mss,
  481. )
  482. }
  483. if count > 255 {
  484. return -2
  485. }
  486. if count == 0 {
  487. count = 1
  488. }
  489. for i := 0; i < count; i++ {
  490. var size int
  491. if len(
  492. buffer,
  493. ) > int(
  494. GFcp.mss,
  495. ) {
  496. size = int(
  497. GFcp.mss,
  498. )
  499. } else {
  500. size = len(
  501. buffer,
  502. )
  503. }
  504. GFcpSeg := GFcp.newSegment(
  505. size,
  506. )
  507. copy(
  508. GFcpSeg.data,
  509. buffer[:size],
  510. )
  511. if GFcp.stream == 0 {
  512. GFcpSeg.frg = uint8(
  513. count - i - 1,
  514. )
  515. } else {
  516. GFcpSeg.frg = 0
  517. }
  518. GFcp.sndQueue = append(
  519. GFcp.sndQueue,
  520. GFcpSeg,
  521. )
  522. buffer = buffer[size:]
  523. }
  524. return 0
  525. }
  526. func (
  527. GFcp *GFCP,
  528. ) updateAck(
  529. rtt int32,
  530. ) {
  531. var rto uint32
  532. if GFcp.rxSrtt == 0 {
  533. GFcp.rxSrtt = rtt
  534. GFcp.rxRttVar = rtt >> 1
  535. } else {
  536. delta := rtt - GFcp.rxSrtt
  537. GFcp.rxSrtt += delta >> 3
  538. if delta < 0 {
  539. delta = -delta
  540. }
  541. if rtt < GFcp.rxSrtt-GFcp.rxRttVar {
  542. GFcp.rxRttVar += (delta - GFcp.rxRttVar) >> 5
  543. } else {
  544. GFcp.rxRttVar += (delta - GFcp.rxRttVar) >> 2
  545. }
  546. }
  547. rto = uint32(
  548. GFcp.rxSrtt,
  549. ) + _imax(
  550. GFcp.interval,
  551. uint32(
  552. GFcp.rxRttVar,
  553. )<<2)
  554. GFcp.rxRto = _ibound(
  555. GFcp.rxMinRto,
  556. rto,
  557. GfcpRtoMax,
  558. )
  559. }
  560. func (
  561. GFcp *GFCP,
  562. ) shrinkBuf() {
  563. if len(
  564. GFcp.SndBuf,
  565. ) > 0 {
  566. GFcpSeg := &GFcp.SndBuf[0]
  567. GFcp.sndUna = GFcpSeg.sn
  568. } else {
  569. GFcp.sndUna = GFcp.sndNxt
  570. }
  571. }
  572. func (
  573. GFcp *GFCP,
  574. ) parseAck(
  575. sn uint32,
  576. ) {
  577. if _itimediff(
  578. sn,
  579. GFcp.sndUna,
  580. ) < 0 || _itimediff(
  581. sn,
  582. GFcp.sndNxt,
  583. ) >= 0 {
  584. return
  585. }
  586. for k := range GFcp.SndBuf {
  587. GFcpSeg := &GFcp.SndBuf[k]
  588. if sn == GFcpSeg.sn {
  589. GFcpSeg.acked = 1
  590. GFcp.delSegment(
  591. GFcpSeg,
  592. )
  593. break
  594. }
  595. if _itimediff(
  596. sn,
  597. GFcpSeg.sn,
  598. ) < 0 {
  599. break
  600. }
  601. }
  602. }
  603. func (
  604. GFcp *GFCP,
  605. ) parseFastack(
  606. sn, ts uint32,
  607. ) {
  608. if _itimediff(
  609. sn,
  610. GFcp.sndUna,
  611. ) < 0 || _itimediff(
  612. sn,
  613. GFcp.sndNxt,
  614. ) >= 0 {
  615. return
  616. }
  617. for k := range GFcp.SndBuf {
  618. GFcpSeg := &GFcp.SndBuf[k]
  619. if _itimediff(
  620. sn,
  621. GFcpSeg.sn,
  622. ) < 0 {
  623. break
  624. } else if sn != GFcpSeg.sn && _itimediff(
  625. GFcpSeg.ts,
  626. ts,
  627. ) <= 0 {
  628. GFcpSeg.fastack++
  629. }
  630. }
  631. }
  632. func (
  633. GFcp *GFCP,
  634. ) parseUna(
  635. una uint32,
  636. ) {
  637. count := 0
  638. for k := range GFcp.SndBuf {
  639. GFcpSeg := &GFcp.SndBuf[k]
  640. if _itimediff(
  641. una,
  642. GFcpSeg.sn,
  643. ) > 0 {
  644. GFcp.delSegment(
  645. GFcpSeg,
  646. )
  647. count++
  648. } else {
  649. break
  650. }
  651. }
  652. if count > 0 {
  653. GFcp.SndBuf = GFcp.removeFront(
  654. GFcp.SndBuf,
  655. count,
  656. )
  657. }
  658. }
  659. func (
  660. GFcp *GFCP,
  661. ) ackPush(
  662. sn,
  663. ts uint32,
  664. ) {
  665. GFcp.acklist = append(
  666. GFcp.acklist,
  667. ackItem{
  668. sn,
  669. ts,
  670. })
  671. }
  672. func (
  673. GFcp *GFCP,
  674. ) parseData(
  675. newGFcpSeg Segment,
  676. ) bool {
  677. sn := newGFcpSeg.sn
  678. if _itimediff(
  679. sn,
  680. GFcp.rcvNxt+GFcp.rcvWnd,
  681. ) >= 0 ||
  682. _itimediff(
  683. sn,
  684. GFcp.rcvNxt,
  685. ) < 0 {
  686. return true
  687. }
  688. n := len(
  689. GFcp.rcvBuf,
  690. ) - 1
  691. insertIdx := 0
  692. repeat := false
  693. for i := n; i >= 0; i-- {
  694. GFcpSeg := &GFcp.rcvBuf[i]
  695. if GFcpSeg.sn == sn {
  696. repeat = true
  697. break
  698. }
  699. if _itimediff(
  700. sn,
  701. GFcpSeg.sn,
  702. ) > 0 {
  703. insertIdx = i + 1
  704. break
  705. }
  706. }
  707. if !repeat {
  708. dataCopy := KxmitBuf.Get().([]byte)[:len(newGFcpSeg.data)]
  709. copy(
  710. dataCopy,
  711. newGFcpSeg.data,
  712. )
  713. newGFcpSeg.data = dataCopy
  714. if insertIdx == n+1 {
  715. GFcp.rcvBuf = append(
  716. GFcp.rcvBuf,
  717. newGFcpSeg,
  718. )
  719. } else {
  720. GFcp.rcvBuf = append(
  721. GFcp.rcvBuf,
  722. Segment{},
  723. )
  724. copy(
  725. GFcp.rcvBuf[insertIdx+1:],
  726. GFcp.rcvBuf[insertIdx:],
  727. )
  728. GFcp.rcvBuf[insertIdx] = newGFcpSeg
  729. }
  730. }
  731. count := 0
  732. for k := range GFcp.rcvBuf {
  733. GFcpSeg := &GFcp.rcvBuf[k]
  734. if GFcpSeg.sn == GFcp.rcvNxt && len(
  735. GFcp.rcvQueue,
  736. ) < int(
  737. GFcp.rcvWnd,
  738. ) {
  739. GFcp.rcvNxt++
  740. count++
  741. } else {
  742. break
  743. }
  744. }
  745. if count > 0 {
  746. GFcp.rcvQueue = append(
  747. GFcp.rcvQueue,
  748. GFcp.rcvBuf[:count]...,
  749. )
  750. GFcp.rcvBuf = GFcp.removeFront(
  751. GFcp.rcvBuf,
  752. count,
  753. )
  754. }
  755. return repeat
  756. }
  757. // Input receives a (low-level) UDP packet, and determinines if
  758. // a full packet has been processsed (not by the FEC algorithm)
  759. func (
  760. GFcp *GFCP,
  761. ) Input(
  762. data []byte,
  763. regular,
  764. ackNoDelay bool,
  765. ) int {
  766. sndUna := GFcp.sndUna
  767. if len(
  768. data,
  769. ) < GfcpOverhead {
  770. return -1
  771. }
  772. var latest uint32
  773. var flag int
  774. var inSegs uint64
  775. for {
  776. var ts,
  777. sn,
  778. length,
  779. una,
  780. conv uint32
  781. var wnd uint16
  782. var cmd,
  783. frg uint8
  784. if len(
  785. data,
  786. ) < int(
  787. GfcpOverhead,
  788. ) {
  789. break
  790. }
  791. data = gfcpDecode32u(
  792. data,
  793. &conv,
  794. )
  795. if conv != GFcp.conv {
  796. return -1
  797. }
  798. data = gfcpDecode8u(
  799. data,
  800. &cmd,
  801. )
  802. data = gfcpDecode8u(
  803. data,
  804. &frg,
  805. )
  806. data = gfcpDecode16u(
  807. data,
  808. &wnd,
  809. )
  810. data = gfcpDecode32u(
  811. data,
  812. &ts,
  813. )
  814. data = gfcpDecode32u(
  815. data,
  816. &sn,
  817. )
  818. data = gfcpDecode32u(
  819. data,
  820. &una,
  821. )
  822. data = gfcpDecode32u(
  823. data,
  824. &length,
  825. )
  826. if len(
  827. data,
  828. ) < int(
  829. length,
  830. ) {
  831. return -2
  832. }
  833. if cmd != GfcpCmdPush && cmd != GfcpCmdAck &&
  834. cmd != GfcpCmdWask && cmd != GfcpCmdWins {
  835. return -3
  836. }
  837. if regular {
  838. GFcp.rmtWnd = uint32(
  839. wnd,
  840. )
  841. }
  842. GFcp.parseUna(
  843. una,
  844. )
  845. GFcp.shrinkBuf()
  846. if cmd == GfcpCmdAck {
  847. GFcp.parseAck(
  848. sn,
  849. )
  850. GFcp.parseFastack(
  851. sn,
  852. ts,
  853. )
  854. flag |= 1
  855. latest = ts
  856. } else if cmd == GfcpCmdPush {
  857. repeat := true
  858. if _itimediff(
  859. sn,
  860. GFcp.rcvNxt+GFcp.rcvWnd,
  861. ) < 0 {
  862. GFcp.ackPush(
  863. sn,
  864. ts,
  865. )
  866. if _itimediff(
  867. sn,
  868. GFcp.rcvNxt,
  869. ) >= 0 {
  870. var GFcpSeg Segment
  871. GFcpSeg.conv = conv
  872. GFcpSeg.cmd = cmd
  873. GFcpSeg.frg = frg
  874. GFcpSeg.wnd = wnd
  875. GFcpSeg.ts = ts
  876. GFcpSeg.sn = sn
  877. GFcpSeg.una = una
  878. GFcpSeg.data = data[:length]
  879. repeat = GFcp.parseData(
  880. GFcpSeg,
  881. )
  882. }
  883. }
  884. if regular && repeat {
  885. atomic.AddUint64(
  886. &DefaultSnsi.GFcpDupSegments,
  887. 1,
  888. )
  889. }
  890. } else if cmd == GfcpCmdWask {
  891. GFcp.probe |= GfcpAskTell
  892. //} else if cmd == GfcpCmdWins {
  893. // XXX(jhj) ??? FUCK YOU CHINKS
  894. } else {
  895. return -3
  896. }
  897. inSegs++
  898. data = data[length:]
  899. }
  900. atomic.AddUint64(
  901. &DefaultSnsi.GFcpInputSegments,
  902. inSegs,
  903. )
  904. if flag != 0 && regular {
  905. current := CurrentMs()
  906. if _itimediff(
  907. current,
  908. latest,
  909. ) >= 0 {
  910. GFcp.updateAck(
  911. _itimediff(
  912. current,
  913. latest,
  914. ),
  915. )
  916. }
  917. }
  918. if GFcp.nocwnd == 0 {
  919. if _itimediff(
  920. GFcp.sndUna,
  921. sndUna,
  922. ) > 0 {
  923. if GFcp.cwnd < GFcp.rmtWnd {
  924. mss := GFcp.mss
  925. if GFcp.cwnd < GFcp.ssthresh {
  926. GFcp.cwnd++
  927. GFcp.incr += mss
  928. } else {
  929. if GFcp.incr < mss {
  930. GFcp.incr = mss
  931. }
  932. GFcp.incr += (mss*mss)/GFcp.incr + (mss / 16)
  933. if (GFcp.cwnd+1)*mss <= GFcp.incr {
  934. GFcp.cwnd++
  935. }
  936. }
  937. if GFcp.cwnd > GFcp.rmtWnd {
  938. GFcp.cwnd = GFcp.rmtWnd
  939. GFcp.incr = GFcp.rmtWnd * mss
  940. }
  941. }
  942. }
  943. }
  944. if ackNoDelay && len(
  945. GFcp.acklist,
  946. ) > 0 {
  947. GFcp.Flush(
  948. true,
  949. )
  950. }
  951. return 0
  952. }
  953. func (
  954. GFcp *GFCP,
  955. ) wndUnused() uint16 {
  956. if len(
  957. GFcp.rcvQueue,
  958. ) < int(GFcp.rcvWnd) {
  959. return uint16(
  960. int(
  961. GFcp.rcvWnd,
  962. ) - len(
  963. GFcp.rcvQueue,
  964. ),
  965. )
  966. }
  967. return 0
  968. }
  969. // Flush ...
  970. func (
  971. GFcp *GFCP,
  972. ) Flush(
  973. ackOnly bool,
  974. ) uint32 {
  975. var GFcpSeg Segment
  976. GFcpSeg.conv = GFcp.conv
  977. GFcpSeg.cmd = GfcpCmdAck
  978. GFcpSeg.wnd = GFcp.wndUnused()
  979. GFcpSeg.una = GFcp.rcvNxt
  980. buffer := GFcp.buffer
  981. ptr := buffer[GFcp.reserved:]
  982. makeSpace := func(
  983. space int,
  984. ) {
  985. size := len(
  986. buffer,
  987. ) - len(
  988. ptr,
  989. )
  990. if size+space > int(
  991. GFcp.mtu,
  992. ) {
  993. GFcp.output(
  994. buffer,
  995. size,
  996. )
  997. ptr = buffer[GFcp.reserved:]
  998. }
  999. }
  1000. FlushBuffer := func() {
  1001. size := len(
  1002. buffer,
  1003. ) - len(
  1004. ptr,
  1005. )
  1006. if size > GFcp.reserved {
  1007. GFcp.output(
  1008. buffer,
  1009. size,
  1010. )
  1011. }
  1012. }
  1013. for i, ack := range GFcp.acklist {
  1014. makeSpace(
  1015. GfcpOverhead,
  1016. )
  1017. if ack.sn >= GFcp.rcvNxt || len(
  1018. GFcp.acklist,
  1019. )-1 == i {
  1020. GFcpSeg.sn,
  1021. GFcpSeg.ts = ack.sn,
  1022. ack.ts
  1023. ptr = GFcpSeg.encode(
  1024. ptr,
  1025. )
  1026. }
  1027. }
  1028. GFcp.acklist = GFcp.acklist[0:0]
  1029. if ackOnly {
  1030. FlushBuffer()
  1031. return GFcp.interval
  1032. }
  1033. if GFcp.rmtWnd == 0 {
  1034. current := CurrentMs()
  1035. if GFcp.probeWait == 0 {
  1036. GFcp.probeWait = GfcpProbeInit
  1037. GFcp.tsProbe = current + GFcp.probeWait
  1038. } else if _itimediff(
  1039. current,
  1040. GFcp.tsProbe,
  1041. ) >= 0 {
  1042. if GFcp.probeWait < GfcpProbeInit {
  1043. GFcp.probeWait = GfcpProbeInit
  1044. }
  1045. GFcp.probeWait += GFcp.probeWait / 2
  1046. if GFcp.probeWait > GfcpProbeLimit {
  1047. GFcp.probeWait = GfcpProbeLimit
  1048. }
  1049. GFcp.tsProbe = current + GFcp.probeWait
  1050. GFcp.probe |= GfcpAskSend
  1051. }
  1052. }
  1053. GFcp.tsProbe = 0
  1054. GFcp.probeWait = 0
  1055. if (GFcp.probe & GfcpAskSend) != 0 {
  1056. GFcpSeg.cmd = GfcpCmdWask
  1057. makeSpace(
  1058. GfcpOverhead,
  1059. )
  1060. ptr = GFcpSeg.encode(
  1061. ptr,
  1062. )
  1063. }
  1064. if (GFcp.probe & GfcpAskTell) != 0 {
  1065. GFcpSeg.cmd = GfcpCmdWins
  1066. makeSpace(
  1067. GfcpOverhead,
  1068. )
  1069. ptr = GFcpSeg.encode(
  1070. ptr,
  1071. )
  1072. }
  1073. GFcp.probe = 0
  1074. cwnd := _imin(
  1075. GFcp.sndWnd,
  1076. GFcp.rmtWnd,
  1077. )
  1078. if GFcp.nocwnd == 0 {
  1079. cwnd = _imin(
  1080. GFcp.cwnd,
  1081. cwnd,
  1082. )
  1083. }
  1084. newSegsCount := 0
  1085. for k := range GFcp.sndQueue {
  1086. if _itimediff(
  1087. GFcp.sndNxt,
  1088. GFcp.sndUna+cwnd,
  1089. ) >= 0 {
  1090. break
  1091. }
  1092. newGFcpSeg := GFcp.sndQueue[k]
  1093. newGFcpSeg.conv = GFcp.conv
  1094. newGFcpSeg.cmd = GfcpCmdPush
  1095. newGFcpSeg.sn = GFcp.sndNxt
  1096. GFcp.SndBuf = append(
  1097. GFcp.SndBuf,
  1098. newGFcpSeg,
  1099. )
  1100. GFcp.sndNxt++
  1101. newSegsCount++
  1102. }
  1103. if newSegsCount > 0 {
  1104. GFcp.sndQueue = GFcp.removeFront(
  1105. GFcp.sndQueue,
  1106. newSegsCount,
  1107. )
  1108. }
  1109. resent := uint32(
  1110. GFcp.fastresend,
  1111. )
  1112. if GFcp.fastresend <= 0 {
  1113. resent = 0xFFFFFFFF
  1114. }
  1115. current := CurrentMs()
  1116. var change,
  1117. lost,
  1118. lostSegs,
  1119. fastGFcpRestransmittedSegments,
  1120. earlyGFcpRestransmittedSegments uint64
  1121. minrto := int32(
  1122. GFcp.interval,
  1123. )
  1124. ref := GFcp.SndBuf[:len(
  1125. GFcp.SndBuf,
  1126. )]
  1127. for k := range ref {
  1128. Segment := &ref[k]
  1129. needsend := false
  1130. if Segment.acked == 1 {
  1131. continue
  1132. }
  1133. if Segment.Kxmit == 0 {
  1134. needsend = true
  1135. Segment.rto = GFcp.rxRto
  1136. Segment.GFcpResendTs = current + Segment.rto
  1137. } else if _itimediff(
  1138. current,
  1139. Segment.GFcpResendTs,
  1140. ) >= 0 {
  1141. needsend = true
  1142. if GFcp.nodelay == 0 {
  1143. Segment.rto += GFcp.rxRto
  1144. } else {
  1145. Segment.rto += GFcp.rxRto / 2
  1146. }
  1147. Segment.GFcpResendTs = current + Segment.rto
  1148. lost++
  1149. lostSegs++
  1150. } else if Segment.fastack >= resent {
  1151. needsend = true
  1152. Segment.fastack = 0
  1153. Segment.rto = GFcp.rxRto
  1154. Segment.GFcpResendTs = current + Segment.rto
  1155. change++
  1156. fastGFcpRestransmittedSegments++
  1157. } else if Segment.fastack > 0 && newSegsCount == 0 {
  1158. needsend = true
  1159. Segment.fastack = 0
  1160. Segment.rto = GFcp.rxRto
  1161. Segment.GFcpResendTs = current + Segment.rto
  1162. change++
  1163. earlyGFcpRestransmittedSegments++
  1164. }
  1165. if needsend {
  1166. current = CurrentMs()
  1167. Segment.Kxmit++
  1168. Segment.ts = current
  1169. Segment.wnd = GFcpSeg.wnd
  1170. Segment.una = GFcpSeg.una
  1171. need := GfcpOverhead + len(
  1172. Segment.data,
  1173. )
  1174. makeSpace(
  1175. need,
  1176. )
  1177. ptr = Segment.encode(
  1178. ptr,
  1179. )
  1180. copy(
  1181. ptr,
  1182. Segment.data,
  1183. )
  1184. ptr = ptr[len(
  1185. Segment.data,
  1186. ):]
  1187. if Segment.Kxmit >= GFcp.deadLink {
  1188. GFcp.state = 0xFFFFFFFF
  1189. }
  1190. }
  1191. if rto := _itimediff(
  1192. Segment.GFcpResendTs,
  1193. current,
  1194. ); rto > 0 && rto < minrto {
  1195. minrto = rto
  1196. }
  1197. }
  1198. FlushBuffer()
  1199. sum := lostSegs
  1200. if lostSegs > 0 {
  1201. atomic.AddUint64(
  1202. &DefaultSnsi.GFcpLostSegments,
  1203. lostSegs,
  1204. )
  1205. }
  1206. if fastGFcpRestransmittedSegments > 0 {
  1207. atomic.AddUint64(
  1208. &DefaultSnsi.FastGFcpRestransmittedSegments,
  1209. fastGFcpRestransmittedSegments,
  1210. )
  1211. sum += fastGFcpRestransmittedSegments
  1212. }
  1213. if earlyGFcpRestransmittedSegments > 0 {
  1214. atomic.AddUint64(
  1215. &DefaultSnsi.EarlyGFcpRestransmittedSegments,
  1216. earlyGFcpRestransmittedSegments,
  1217. )
  1218. sum += earlyGFcpRestransmittedSegments
  1219. }
  1220. if sum > 0 {
  1221. atomic.AddUint64(
  1222. &DefaultSnsi.GFcpRestransmittedSegments,
  1223. sum,
  1224. )
  1225. }
  1226. if GFcp.nocwnd == 0 {
  1227. if change > 0 {
  1228. inflight := GFcp.sndNxt - GFcp.sndUna
  1229. GFcp.ssthresh = inflight / 2
  1230. if GFcp.ssthresh < GfcpThreshMin {
  1231. GFcp.ssthresh = GfcpThreshMin
  1232. }
  1233. GFcp.cwnd = GFcp.ssthresh + resent
  1234. GFcp.incr = GFcp.cwnd * GFcp.mss
  1235. }
  1236. if lost > 0 {
  1237. GFcp.ssthresh = cwnd / 2
  1238. if GFcp.ssthresh < GfcpThreshMin {
  1239. GFcp.ssthresh = GfcpThreshMin
  1240. }
  1241. GFcp.cwnd = 1
  1242. GFcp.incr = GFcp.mss
  1243. }
  1244. if GFcp.cwnd < 1 {
  1245. GFcp.cwnd = 1
  1246. GFcp.incr = GFcp.mss
  1247. }
  1248. }
  1249. return uint32(
  1250. minrto,
  1251. )
  1252. }
  1253. // Update is called repeatedly, 10ms to 100ms, queried via gfcp_check
  1254. // without gfcp_input or _send executing, returning timestamp in ms.
  1255. func (
  1256. GFcp *GFCP,
  1257. ) Update() {
  1258. var slap int32
  1259. current := CurrentMs()
  1260. if GFcp.updated == 0 {
  1261. GFcp.updated = 1
  1262. GFcp.tsFlush = current
  1263. }
  1264. slap = _itimediff(
  1265. current,
  1266. GFcp.tsFlush,
  1267. )
  1268. if slap >= 10000 || slap < -10000 {
  1269. GFcp.tsFlush = current
  1270. slap = 0
  1271. }
  1272. if slap >= 0 {
  1273. GFcp.tsFlush += GFcp.interval
  1274. if _itimediff(
  1275. current,
  1276. GFcp.tsFlush,
  1277. ) >= 0 {
  1278. GFcp.tsFlush = current + GFcp.interval
  1279. }
  1280. GFcp.Flush(
  1281. false,
  1282. )
  1283. }
  1284. }
  1285. // Check function helps determine when to invoke an gfcp_update.
  1286. // It returns when you should invoke gfcp_update, in milliseconds,
  1287. // if there is no gfcp_input or _send calling. You may repeatdly
  1288. // call gfcp_update instead of update, to reduce most unnacessary
  1289. // gfcp_update invocations. This function may be used to schedule
  1290. // gfcp_updates, when implementing an epoll-like mechanism, or for
  1291. // optimizing an gfcp_update loop handling massive GFcp connections.
  1292. func (
  1293. GFcp *GFCP,
  1294. ) Check() uint32 {
  1295. current := CurrentMs()
  1296. tsFlush := GFcp.tsFlush
  1297. tmFlush := int32(
  1298. math.MaxInt32,
  1299. )
  1300. tmPacket := int32(
  1301. math.MaxInt32,
  1302. )
  1303. minimal := uint32(
  1304. 0,
  1305. )
  1306. if GFcp.updated == 0 {
  1307. return current
  1308. }
  1309. if _itimediff(
  1310. current,
  1311. tsFlush,
  1312. ) >= 10000 ||
  1313. _itimediff(
  1314. current,
  1315. tsFlush,
  1316. ) < -10000 {
  1317. tsFlush = current
  1318. }
  1319. if _itimediff(
  1320. current,
  1321. tsFlush,
  1322. ) >= 0 {
  1323. return current
  1324. }
  1325. tmFlush = _itimediff(
  1326. tsFlush,
  1327. current,
  1328. )
  1329. for k := range GFcp.SndBuf {
  1330. GFcpSeg := &GFcp.SndBuf[k]
  1331. diff := _itimediff(
  1332. GFcpSeg.GFcpResendTs,
  1333. current,
  1334. )
  1335. if diff <= 0 {
  1336. return current
  1337. }
  1338. if diff < tmPacket {
  1339. tmPacket = diff
  1340. }
  1341. }
  1342. minimal = uint32(
  1343. tmPacket,
  1344. )
  1345. if tmPacket >= tmFlush {
  1346. minimal = uint32(
  1347. tmFlush,
  1348. )
  1349. }
  1350. if minimal >= GFcp.interval {
  1351. minimal = GFcp.interval
  1352. }
  1353. return current + minimal
  1354. }
  1355. // SetMtu changes MTU size.
  1356. func (
  1357. GFcp *GFCP,
  1358. ) SetMtu(
  1359. mtu int,
  1360. ) int {
  1361. if mtu < 50 || mtu < GfcpOverhead {
  1362. return -1
  1363. }
  1364. if GFcp.reserved >= int(
  1365. GFcp.mtu-GfcpOverhead,
  1366. ) || GFcp.reserved < 0 {
  1367. return -1
  1368. }
  1369. buffer := make(
  1370. []byte,
  1371. mtu,
  1372. )
  1373. /*if buffer == nil {
  1374. return -2
  1375. }*/ // XXX(jhj): buffer can't be nil?
  1376. GFcp.mtu = uint32(
  1377. mtu,
  1378. )
  1379. GFcp.mss = GFcp.mtu - GfcpOverhead - uint32(
  1380. GFcp.reserved,
  1381. )
  1382. GFcp.buffer = buffer
  1383. return 0
  1384. }
  1385. // NoDelay options:
  1386. // * fastest: gfcp_nodelay(GFcp, 1, 20, 2, 1)
  1387. // * nodelay: 0: disable (default), 1: enable
  1388. // * interval: internal update timer interval in milliseconds, defaults to 100ms
  1389. // * resend: 0: disable fast resends (default), 1: enable fast resends
  1390. // * nc: 0: normal congestion control (default), 1: disable congestion control
  1391. func (
  1392. GFcp *GFCP,
  1393. ) NoDelay(
  1394. nodelay,
  1395. interval,
  1396. resend,
  1397. nc int,
  1398. ) int {
  1399. if nodelay >= 0 {
  1400. GFcp.nodelay = uint32(
  1401. nodelay,
  1402. )
  1403. if nodelay != 0 {
  1404. GFcp.rxMinRto = GfcpRtoNdl
  1405. } else {
  1406. GFcp.rxMinRto = GfcpRtoMin
  1407. }
  1408. }
  1409. if interval >= 0 {
  1410. if interval > 5000 {
  1411. interval = 5000
  1412. } else if interval < 10 {
  1413. interval = 10
  1414. }
  1415. GFcp.interval = uint32(
  1416. interval,
  1417. )
  1418. }
  1419. if resend >= 0 {
  1420. GFcp.fastresend = int32(
  1421. resend,
  1422. )
  1423. }
  1424. if nc >= 0 {
  1425. GFcp.nocwnd = int32(
  1426. nc,
  1427. )
  1428. }
  1429. return 0
  1430. }
  1431. // WndSize sets maximum window size (efaults: sndwnd=32 and rcvwnd=32)
  1432. func (
  1433. GFcp *GFCP,
  1434. ) WndSize(
  1435. sndwnd,
  1436. rcvwnd int,
  1437. ) int {
  1438. if sndwnd > 0 {
  1439. GFcp.sndWnd = uint32(
  1440. sndwnd,
  1441. )
  1442. }
  1443. if rcvwnd > 0 {
  1444. GFcp.rcvWnd = uint32(
  1445. rcvwnd,
  1446. )
  1447. }
  1448. return 0
  1449. }
  1450. // WaitSnd shows how many packets are queued to be sent
  1451. func (
  1452. GFcp *GFCP,
  1453. ) WaitSnd() int {
  1454. return len(
  1455. GFcp.SndBuf,
  1456. ) + len(
  1457. GFcp.sndQueue,
  1458. )
  1459. }
  1460. func (
  1461. GFcp *GFCP,
  1462. ) removeFront(
  1463. q []Segment,
  1464. n int,
  1465. ) []Segment {
  1466. if n > cap(
  1467. q,
  1468. )/2 {
  1469. newn := copy(
  1470. q,
  1471. q[n:],
  1472. )
  1473. return q[:newn]
  1474. }
  1475. return q[n:]
  1476. }
  1477. func init() {
  1478. debug.SetGCPercent(
  1479. 180,
  1480. )
  1481. gfcpLegal.RegisterLicense(
  1482. "\nThe MIT License (MIT)\n\nCopyright © 2015 Daniel Fu <daniel820313@gmail.com>.\nCopyright © 2019 Loki 'l0k18' Verloren <stalker.loki@protonmail.ch>.\nCopyright © 2021 Gridfinity, LLC. <admin@gridfinity.com>.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including, without limitation, the rights\nto use, copy, modify, merge, publish, distribute, sub-license, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice, and this permission notice, shall be\nincluded in all copies, or substantial portions, of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING, BUT NOT LIMITED TO, THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF, OR IN CONNECTION WITH THE SOFTWARE, OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n",
  1483. )
  1484. }