-
Notifications
You must be signed in to change notification settings - Fork 43
Expand file tree
/
Copy pathp3turtle.cc
More file actions
2541 lines (2075 loc) · 87.3 KB
/
p3turtle.cc
File metadata and controls
2541 lines (2075 loc) · 87.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*******************************************************************************
* libretroshare/src/turtle: p3turtle.cc *
* *
* libretroshare: retroshare core library *
* *
* Copyright (C) 2009-2018 Cyril Soler <csoler@users.sourceforge.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
//#define P3TURTLE_DEBUG
#include <unistd.h>
#include <stdexcept>
#include <stdlib.h>
#include <assert.h>
#include <iostream>
#include <cerrno>
#include <cmath>
#include <cstdio>
#include "rsserver/p3face.h"
#include "crypto/rscrypto.h"
#include "pqi/authssl.h"
#include "pqi/p3linkmgr.h"
#include "retroshare/rspeers.h"
#include <retroshare/rsgxsdistsync.h>
#include "ft/ftserver.h"
#include "ft/ftdatamultiplex.h"
#include "ft/ftcontroller.h"
#include "p3turtle.h"
#include "util/cxx17retrocompat.h"
#include "util/rsdebug.h"
#include "util/rsprint.h"
#include "util/rsrandom.h"
#include "pqi/pqinetwork.h"
#ifdef TUNNEL_STATISTICS
static std::vector<int> TS_tunnel_length(8,0) ;
static std::map<TurtleFileHash, std::vector<std::pair<rstime_t,TurtleTunnelRequestId> > > TS_request_time_stamps ;
static std::map<TurtleTunnelRequestId, std::vector<rstime_t> > TS_request_bounces ;
void TS_dumpState() ;
#endif
#define TURTLE_DEBUG() std::cerr << time(NULL) << " : TURTLE : " << __FUNCTION__ << " : "
#define TURTLE_ERROR() std::cerr << "(EE) TURTLE ERROR : "
// These number may be quite important. I setup them with sensible values, but
// an in-depth test would be better to get an idea of what the ideal values
// could ever be.
//
// update of 14-03-11:
// - I raised the cache time for tunnel requests. This avoids inconsistencies such as:
// * tunnel requests bouncing back while the original request is not in the cache anymore
// * special case of this for own file transfer: an outgoing tunnel is built with no end.
// - changed tunnel speed estimate time lapse to 5 secs. Too small a value favors high variations.
// - the max number of tunnel requests per second is now enforced. It was before defaulting to
// QUEUE_LENGTH*0.1, meaning 0.5. I set it to 0.1.
//
// update of 19-04-12:
// - The total number of TR per second emmited from self will be MAX_TUNNEL_REQS_PER_SECOND / TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS = 0.5
// - I updated forward probabilities to higher values, and min them to 1/nb_connected_friends to prevent blocking tunnels.
//
static const rstime_t TUNNEL_REQUESTS_LIFE_TIME = 600 ; /// life time for tunnel requests in the cache.
static const rstime_t TUNNEL_REQUESTS_RESULT_TIME = 20 ; /// maximum time during which we process/forward results for known tunnel requests
static const rstime_t SEARCH_REQUESTS_LIFE_TIME = 600 ; /// life time for search requests in the cache
static const rstime_t SEARCH_REQUESTS_RESULT_TIME = 20 ; /// maximum time during which we process/forward results for known search requests
static const rstime_t REGULAR_TUNNEL_DIGGING_TIME = 300 ; /// maximum interval between two tunnel digging campaigns.
static const rstime_t MAXIMUM_TUNNEL_IDLE_TIME = 60 ; /// maximum life time of an unused tunnel.
static const rstime_t EMPTY_TUNNELS_DIGGING_TIME = 50 ; /// look into tunnels regularly every 50 sec.
static const rstime_t TUNNEL_SPEED_ESTIMATE_LAPSE = 5 ; /// estimate tunnel speed every 5 seconds
static const rstime_t TUNNEL_CLEANING_LAPS_TIME = 10 ; /// clean tunnels every 10 secs
static const rstime_t TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS = 2 ; /// Tunnel management calls every 2 secs.
static const uint32_t MAX_TUNNEL_REQS_PER_SECOND = 1 ; /// maximum number of tunnel requests issued per second. Was 0.5 before
static const uint32_t MAX_ALLOWED_SR_IN_CACHE = 120 ; /// maximum number of search requests allowed in cache. That makes 2 per sec.
static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_FILES =5000 ; /// maximum number of search results forwarded back to the source.
static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT = 100 ; /// default maximum number of search results forwarded back source.
static const float depth_peer_probability[7] = { 1.0f,0.99f,0.9f,0.7f,0.6f,0.5,0.4f } ;
static const int TUNNEL_REQUEST_PACKET_SIZE = 50 ;
static const int MAX_TR_FORWARD_PER_SEC = 20 ;
static const int MAX_TR_FORWARD_PER_SEC_UPPER_LIMIT = 100 ;
static const int MAX_TR_FORWARD_PER_SEC_LOWER_LIMIT = 10 ;
static const int DISTANCE_SQUEEZING_POWER = 8 ;
#define HEX_PRINT(a) std::hex << a << std::dec
p3turtle::p3turtle(p3ServiceControl *sc,p3LinkMgr *lm)
:p3Service(), p3Config(), mServiceControl(sc), mLinkMgr(lm), mTurtleMtx("p3turtle")
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
_own_id = sc->getOwnId() ;
if(_own_id.isNull())
{
std::cerr << "OwnId is null, as returned by the p3ServiceControl object in turtle router. Can't go on!" << std::endl;
exit(-1) ;
}
_turtle_routing_enabled = true ;
_turtle_routing_session_enabled = true;
_random_bias = RSRandom::random_u32() ;
_serialiser = new RsTurtleSerialiser() ;
addSerialType(_serialiser);
_last_clean_time = 0 ;
_last_tunnel_management_time = 0 ;
_last_tunnel_campaign_time = 0 ;
_last_tunnel_speed_estimate_time = 0 ;
_traffic_info.reset() ;
_max_tr_up_rate = MAX_TR_FORWARD_PER_SEC ;
_service_type = getServiceInfo().mServiceType ;
}
const std::string TURTLE_APP_NAME = "turtle";
const uint16_t TURTLE_APP_MAJOR_VERSION = 1;
const uint16_t TURTLE_APP_MINOR_VERSION = 0;
const uint16_t TURTLE_MIN_MAJOR_VERSION = 1;
const uint16_t TURTLE_MIN_MINOR_VERSION = 0;
RsServiceInfo p3turtle::getServiceInfo()
{
return RsServiceInfo(RS_SERVICE_TYPE_TURTLE,
TURTLE_APP_NAME,
TURTLE_APP_MAJOR_VERSION,
TURTLE_APP_MINOR_VERSION,
TURTLE_MIN_MAJOR_VERSION,
TURTLE_MIN_MINOR_VERSION);
}
void p3turtle::getItemNames(std::map<uint8_t,std::string>& names) const
{
names.clear();
names[RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST ] = "Filename substring search request";
names[RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST ] = "Generic search request";
names[RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT ] = "File search result";
names[RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT ] = "Generic search result";
names[RS_TURTLE_SUBTYPE_OPEN_TUNNEL ] = "Tunnel request";
names[RS_TURTLE_SUBTYPE_TUNNEL_OK ] = "Tunnel response";
names[RS_TURTLE_SUBTYPE_FILE_REQUEST ] = "Data request";
names[RS_TURTLE_SUBTYPE_FILE_DATA ] = "Data chunk";
names[RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST ] = "Filename RegExp search request";
names[RS_TURTLE_SUBTYPE_GENERIC_DATA ] = "Generic data";
names[RS_TURTLE_SUBTYPE_FILE_MAP ] = "Chunk map";
names[RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST ] = "Chunk map request";
names[RS_TURTLE_SUBTYPE_CHUNK_CRC ] = "Chunk CRC";
names[RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST ] = "Chunk CRC request";
}
void p3turtle::setEnabled(bool b)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
_turtle_routing_enabled = b;
if(b)
std::cerr << "Enabling turtle routing" << std::endl;
else
std::cerr << "Disabling turtle routing" << std::endl;
IndicateConfigChanged() ;
}
bool p3turtle::enabled() const
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
return _turtle_routing_enabled ;
}
void p3turtle::setSessionEnabled(bool b)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
_turtle_routing_session_enabled = b;
if(b)
std::cerr << "Enabling turtle routing for this Session" << std::endl;
else
std::cerr << "Disabling turtle routing for this Session" << std::endl;
}
bool p3turtle::sessionEnabled() const
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
return _turtle_routing_session_enabled ;
}
int p3turtle::tick()
{
// Handle tunnel trafic
//
handleIncoming(); // handle incoming packets
rstime_t now = time(NULL) ;
#ifdef TUNNEL_STATISTICS
static rstime_t last_now = now ;
if(now - last_now > 2)
std::cerr << "******************* WARNING: now - last_now = " << now - last_now << std::endl;
last_now = now ;
#endif
bool should_autowash,should_estimatespeed ;
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
should_autowash = now > TUNNEL_CLEANING_LAPS_TIME+_last_clean_time ;
should_estimatespeed = now >= TUNNEL_SPEED_ESTIMATE_LAPSE + _last_tunnel_speed_estimate_time ;
}
// Tunnel management:
// - we digg new tunnels at least every 5 min (300 sec).
// - we digg new tunnels each time a new peer connects
// - we digg new tunnels each time a new hash is asked for
//
if(now >= _last_tunnel_management_time+TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS) // call every second
{
#ifdef P3TURTLE_DEBUG
std::cerr << "Calling tunnel management." << std::endl ;
#endif
if(_turtle_routing_enabled && _turtle_routing_session_enabled)
manageTunnels() ;
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
_last_tunnel_management_time = now ;
// Update traffic statistics. The constants are important: they allow a smooth variation of the
// traffic speed, which is used to moderate tunnel requests statistics.
//
_traffic_info = _traffic_info*0.9 + _traffic_info_buffer* (0.1 / (float)TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS) ;
_traffic_info_buffer.reset() ;
}
}
// Clean every 10 sec.
//
if(should_autowash)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "Calling autowash." << std::endl ;
#endif
autoWash() ; // clean old/unused tunnels and file hashes, as well as search and tunnel requests.
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
_last_clean_time = now ;
}
if(should_estimatespeed)
{
estimateTunnelSpeeds() ;
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
_last_tunnel_speed_estimate_time = now ;
}
#ifdef TUNNEL_STATISTICS
// Dump state for debugging, every 20 sec.
//
static rstime_t TS_last_dump = time(NULL) ;
if(now > 20+TS_last_dump)
{
TS_last_dump = now ;
TS_dumpState() ;
}
#endif
#ifdef P3TURTLE_DEBUG
// Dump state for debugging, every 20 sec.
//
static rstime_t last_dump = time(NULL) ;
if(now > 20+last_dump)
{
last_dump = now ;
dumpState() ;
}
#endif
return 0 ;
}
// -----------------------------------------------------------------------------------//
// ------------------------------ Tunnel maintenance. ------------------------------ //
// -----------------------------------------------------------------------------------//
//
// adds a virtual peer to the list that is communicated ot ftController.
//
void p3turtle::locked_addDistantPeer(const TurtleFileHash&,TurtleTunnelId tid)
{
unsigned char tmp[RsPeerId::SIZE_IN_BYTES] ;
memset(tmp,0,RsPeerId::SIZE_IN_BYTES) ;
assert(sizeof(tid) == 4) ;
for(int i=0;i<4;++i)
tmp[i] = uint8_t( (tid >> ((3-i)*8)) & 0xff ) ;
RsPeerId virtual_peer_id(tmp) ;
_virtual_peers[virtual_peer_id] = tid ;
#ifdef P3TURTLE_DEBUG
assert(_local_tunnels.find(tid)!=_local_tunnels.end()) ;
#endif
_local_tunnels[tid].vpid = virtual_peer_id ;
}
void p3turtle::getSourceVirtualPeersList(const TurtleFileHash& hash,std::list<pqipeer>& list)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
list.clear() ;
std::map<TurtleFileHash,TurtleHashInfo>::const_iterator it = _incoming_file_hashes.find(hash) ;
if(it != _incoming_file_hashes.end())
for(uint32_t i=0;i<it->second.tunnels.size();++i)
{
std::map<TurtleTunnelId,TurtleTunnel>::const_iterator it2 = _local_tunnels.find( it->second.tunnels[i] ) ;
if(it2 != _local_tunnels.end())
{
pqipeer vp ;
vp.id = it2->second.vpid ;
vp.name = "Virtual (distant) peer" ;
vp.state = RS_PEER_S_CONNECTED ;
vp.actions = RS_PEER_CONNECTED ;
list.push_back(vp) ;
}
else
std::cerr << "(EE) getSourceVirtualPeersList(): no tunnels for incoming file hash " << hash << ": weird!"<< std::endl;
}
}
// This method handles digging new tunnels as needed.
// New tunnels are dug when:
// - new peers have connected. The resulting tunnels should be checked against doubling.
// - new hashes are submitted for handling.
//
class hashPairComparator
{
public:
virtual bool operator()(const std::pair<TurtleFileHash,rstime_t>& p1,const std::pair<TurtleFileHash,rstime_t>& p2) const
{
return p1.second < p2.second ;
}
};
void p3turtle::manageTunnels()
{
// Collect hashes for which tunnel digging is necessary / recommended. Hashes get in the list for two reasons:
// - the hash has no tunnel -> tunnel digging every EMPTY_TUNNELS_DIGGING_TIME seconds
// - the hash hasn't been tunneled for more than REGULAR_TUNNEL_DIGGING_TIME seconds, even if downloading.
//
// Candidate hashes are sorted, by olderness. The older gets tunneled first. At most MAX_TUNNEL_REQS_PER_SECOND are
// treated at once, as this method is called every second.
// Note: Because REGULAR_TUNNEL_DIGGING_TIME is larger than EMPTY_TUNNELS_DIGGING_TIME, files being downloaded get
// re-tunneled in priority. As this happens less, they don't obliterate tunneling for files that have no tunnels yet.
std::vector<std::pair<TurtleFileHash,rstime_t> > hashes_to_digg ;
rstime_t now = time(NULL) ;
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
// digg new tunnels if no tunnels are available and force digg new tunnels at regular (large) interval
//
for(std::map<TurtleFileHash,TurtleHashInfo>::const_iterator it(_incoming_file_hashes.begin());it!=_incoming_file_hashes.end();++it)
{
// get total tunnel speed.
//
uint32_t total_speed = 0 ;
for(uint32_t i=0;i<it->second.tunnels.size();++i)
total_speed += _local_tunnels[it->second.tunnels[i]].speed_Bps ;
static const float grow_speed = 1.0f ; // speed at which the time increases.
float tunnel_keeping_factor = (std::max(1.0f,(float)total_speed/(float)(50*1024)) - 1.0f)*grow_speed + 1.0f ;
#ifdef P3TURTLE_DEBUG
std::cerr << "Total speed = " << total_speed << ", tunel factor = " << tunnel_keeping_factor << " new time = " << rstime_t(REGULAR_TUNNEL_DIGGING_TIME*tunnel_keeping_factor) << std::endl;
#endif
if( (it->second.tunnels.empty() && now >= it->second.last_digg_time+EMPTY_TUNNELS_DIGGING_TIME)
|| (it->second.use_aggressive_mode && now >= it->second.last_digg_time + rstime_t(REGULAR_TUNNEL_DIGGING_TIME*tunnel_keeping_factor)))
{
#ifdef P3TURTLE_DEBUG
std::cerr << "pushed hash " << it->first << ", for digging. Old = " << now - it->second.last_digg_time << std::endl;
#endif
hashes_to_digg.push_back(std::pair<TurtleFileHash,rstime_t>(it->first,it->second.last_digg_time)) ;
}
}
}
#ifdef TUNNEL_STATISTICS
std::cerr << hashes_to_digg.size() << " hashes candidate for tunnel digging." << std::endl;
#endif
std::sort(hashes_to_digg.begin(),hashes_to_digg.end(),hashPairComparator()) ;
for(unsigned int i=0;i<MAX_TUNNEL_REQS_PER_SECOND && i<hashes_to_digg.size();++i)// Digg at most n tunnels per second.
{
#ifdef TUNNEL_STATISTICS
std::cerr << "!!!!!!!!!!!! Digging for " << hashes_to_digg[i].first << std::endl;
#endif
diggTunnel(hashes_to_digg[i].first) ;
}
}
void p3turtle::estimateTunnelSpeeds()
{
RsStackMutex stack(mTurtleMtx) ;
for(std::map<TurtleTunnelId,TurtleTunnel>::iterator it(_local_tunnels.begin());it!=_local_tunnels.end();++it)
{
TurtleTunnel& tunnel(it->second) ;
float speed_estimate = tunnel.transfered_bytes / float(TUNNEL_SPEED_ESTIMATE_LAPSE) ;
tunnel.speed_Bps = 0.75*tunnel.speed_Bps + 0.25*speed_estimate ;
tunnel.transfered_bytes = 0 ;
}
}
void p3turtle::autoWash()
{
#ifdef P3TURTLE_DEBUG
std::cerr << " In autowash." << std::endl ;
#endif
// Remove hashes that are marked as such.
//
std::vector<std::pair<RsTurtleClientService*,std::pair<TurtleFileHash,TurtleVirtualPeerId> > > services_vpids_to_remove ;
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
for(std::set<RsFileHash>::const_iterator hit(_hashes_to_remove.begin());hit!=_hashes_to_remove.end();++hit)
{
std::map<TurtleFileHash,TurtleHashInfo>::iterator it(_incoming_file_hashes.find(*hit)) ;
if(it == _incoming_file_hashes.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: asked to stop monitoring file hash " << *hit << ", but this hash is actually not handled by the turtle router." << std::endl ;
#endif
continue ;
}
// copy the list of tunnels to remove.
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: stopping monitoring for file hash " << *hit << ", and closing " << it->second.tunnels.size() << " tunnels (" ;
#endif
std::vector<TurtleTunnelId> tunnels_to_remove ;
for(std::vector<TurtleTunnelId>::const_iterator it2(it->second.tunnels.begin());it2!=it->second.tunnels.end();++it2)
{
#ifdef P3TURTLE_DEBUG
std::cerr << HEX_PRINT(*it2) << "," ;
#endif
tunnels_to_remove.push_back(*it2) ;
}
#ifdef P3TURTLE_DEBUG
std::cerr << ")" << std::endl ;
#endif
for(unsigned int k=0;k<tunnels_to_remove.size();++k)
locked_closeTunnel(tunnels_to_remove[k],services_vpids_to_remove) ;
_incoming_file_hashes.erase(it) ;
}
_hashes_to_remove.clear() ;
}
// look for tunnels and stored temporary info that have not been used for a while.
rstime_t now = time(NULL) ;
// Search requests
//
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
for(std::map<TurtleSearchRequestId,TurtleSearchRequestInfo>::iterator it(_search_requests_origins.begin());it!=_search_requests_origins.end();)
if(now > (rstime_t)(it->second.time_stamp + SEARCH_REQUESTS_LIFE_TIME))
{
#ifdef P3TURTLE_DEBUG
std::cerr << " removed search request " << HEX_PRINT(it->first) << ", timeout." << std::endl ;
#endif
std::map<TurtleSearchRequestId,TurtleSearchRequestInfo>::iterator tmp(it) ;
++tmp ;
_search_requests_origins.erase(it) ;
it = tmp ;
}
else
++it;
}
// Tunnel requests
//
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
for(std::map<TurtleTunnelRequestId,TurtleTunnelRequestInfo>::iterator it(_tunnel_requests_origins.begin());it!=_tunnel_requests_origins.end();)
if(now > (rstime_t)(it->second.time_stamp + TUNNEL_REQUESTS_LIFE_TIME))
{
#ifdef P3TURTLE_DEBUG
std::cerr << " removed tunnel request " << HEX_PRINT(it->first) << ", timeout." << std::endl ;
#endif
std::map<TurtleTunnelRequestId,TurtleTunnelRequestInfo>::iterator tmp(it) ;
++tmp ;
_tunnel_requests_origins.erase(it) ;
it = tmp ;
}
else
++it ;
}
// Tunnels.
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
std::vector<TurtleTunnelId> tunnels_to_close ;
for(std::map<TurtleTunnelId,TurtleTunnel>::iterator it(_local_tunnels.begin());it!=_local_tunnels.end();++it)
if(now > (rstime_t)(it->second.time_stamp + MAXIMUM_TUNNEL_IDLE_TIME))
{
#ifdef P3TURTLE_DEBUG
std::cerr << " removing tunnel " << HEX_PRINT(it->first) << ": timeout." << std::endl ;
#endif
tunnels_to_close.push_back(it->first) ;
}
for(unsigned int i=0;i<tunnels_to_close.size();++i)
locked_closeTunnel(tunnels_to_close[i],services_vpids_to_remove) ;
}
// Now remove all the virtual peers ids at the client services. Off mutex!
//
for(uint32_t i=0;i<services_vpids_to_remove.size();++i)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " removing virtual peer id " << services_vpids_to_remove[i].second.second << " for service " << services_vpids_to_remove[i].first <<", for hash " << services_vpids_to_remove[i].second.first << std::endl ;
#endif
services_vpids_to_remove[i].first->removeVirtualPeer(services_vpids_to_remove[i].second.first,services_vpids_to_remove[i].second.second) ;
}
}
void p3turtle::forceReDiggTunnels(const TurtleFileHash& hash)
{
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
if( _incoming_file_hashes.find(hash) == _incoming_file_hashes.end())
{
std::cerr << "(EE) p3turtle::forceReDiggTunnels(): hash " << hash << " is not currently handled by turtle." << std::endl;
return ;
}
}
diggTunnel(hash) ;
}
void p3turtle::locked_closeTunnel(TurtleTunnelId tid,std::vector<std::pair<RsTurtleClientService*,std::pair<TurtleFileHash,TurtleVirtualPeerId> > >& sources_to_remove)
{
// This is closing a given tunnel, removing it from file sources, and from the list of tunnels of its
// corresponding file hash. In the original turtle4privacy paradigm, they also send back and forward
// tunnel closing commands. In our case, this is not necessary, because if a tunnel is closed somewhere, its
// source is not going to be used and the tunnel will eventually disappear.
//
std::map<TurtleTunnelId,TurtleTunnel>::iterator it(_local_tunnels.find(tid)) ;
if(it == _local_tunnels.end())
{
std::cerr << "p3turtle: was asked to close tunnel " << reinterpret_cast<void*>(tid) << ", which actually doesn't exist." << std::endl ;
return ;
}
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: Closing tunnel " << HEX_PRINT(tid) << std::endl ;
#endif
if(it->second.local_src == _own_id) // this is a starting tunnel. We thus remove
// - the virtual peer from the vpid list
// - the tunnel id from the file hash
// - the virtual peer from the file sources in the file transfer controller.
{
TurtleTunnelId tid = it->first ;
TurtleVirtualPeerId vpid = it->second.vpid ;
TurtleFileHash hash = it->second.hash ;
#ifdef P3TURTLE_DEBUG
std::cerr << " Tunnel is a starting point. Also removing:" << std::endl ;
std::cerr << " Virtual Peer Id " << vpid << std::endl ;
std::cerr << " Associated file source." << std::endl ;
#endif
std::pair<TurtleFileHash,TurtleVirtualPeerId> hash_vpid(hash,vpid) ;
// Let's be cautious. Normally we should never be here without consistent information,
// but still, this happens, rarely.
//
if(_virtual_peers.find(vpid) != _virtual_peers.end())
_virtual_peers.erase(_virtual_peers.find(vpid)) ;
std::map<TurtleFileHash,TurtleHashInfo>::iterator it(_incoming_file_hashes.find(hash)) ;
if(it != _incoming_file_hashes.end())
{
std::vector<TurtleTunnelId>& tunnels(it->second.tunnels) ;
// Remove tunnel id from it's corresponding hash. For security we
// go through the whole tab, although the tunnel id should only be listed once
// in this tab.
//
for(unsigned int i=0;i<tunnels.size();)
if(tunnels[i] == tid)
{
tunnels[i] = tunnels.back() ;
tunnels.pop_back() ;
}
else
++i ;
sources_to_remove.push_back(std::pair<RsTurtleClientService*,std::pair<TurtleFileHash,TurtleVirtualPeerId> >(it->second.service,hash_vpid)) ;
}
}
else if(it->second.local_dst == _own_id) // This is a ending tunnel. We also remove the virtual peer id
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Tunnel is a ending point. Also removing associated outgoing hash." ;
#endif
std::map<TurtleTunnelId,RsTurtleClientService*>::iterator itHash = _outgoing_tunnel_client_services.find(tid);
if(itHash != _outgoing_tunnel_client_services.end())
{
TurtleVirtualPeerId vpid = it->second.vpid ;
TurtleFileHash hash = it->second.hash ;
std::pair<TurtleFileHash,TurtleVirtualPeerId> hash_vpid(hash,vpid) ;
sources_to_remove.push_back(std::pair<RsTurtleClientService*,std::pair<TurtleFileHash,TurtleVirtualPeerId> >(itHash->second,hash_vpid)) ;
_outgoing_tunnel_client_services.erase(itHash) ;
// Also remove the associated virtual peer
//
if(_virtual_peers.find(vpid) != _virtual_peers.end())
_virtual_peers.erase(_virtual_peers.find(vpid)) ;
}
}
_local_tunnels.erase(it) ;
}
void p3turtle::stopMonitoringTunnels(const RsFileHash& hash)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: Marking hash " << hash << " to be removed during autowash." << std::endl ;
#endif
// We don't do the deletion in this process, because it can cause a race with tunnel management.
_hashes_to_remove.insert(hash) ;
}
// -----------------------------------------------------------------------------------//
// -------------------------------- Config functions ------------------------------ //
// -----------------------------------------------------------------------------------//
//
RsSerialiser *p3turtle::setupSerialiser()
{
RsSerialiser *rss = new RsSerialiser ;
rss->addSerialType(new RsTurtleSerialiser) ;
rss->addSerialType(new RsGeneralConfigSerialiser());
return rss ;
}
bool p3turtle::saveList(bool& cleanup, std::list<RsItem*>& lst)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: saving list..." << std::endl ;
#endif
cleanup = true ;
RsConfigKeyValueSet *vitem = new RsConfigKeyValueSet ;
RsTlvKeyValue kv;
kv.key = "TURTLE_CONFIG_MAX_TR_RATE" ;
rs_sprintf(kv.value, "%g", _max_tr_up_rate);
vitem->tlvkvs.pairs.push_back(kv) ;
kv.key = "TURTLE_ENABLED" ;
kv.value = _turtle_routing_enabled?"TRUE":"FALSE" ;
vitem->tlvkvs.pairs.push_back(kv) ;
lst.push_back(vitem) ;
return true ;
}
bool p3turtle::loadList(std::list<RsItem*>& load)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: loading list..." << std::endl ;
#endif
for(std::list<RsItem*>::const_iterator it(load.begin());it!=load.end();++it)
{
RsConfigKeyValueSet *vitem = dynamic_cast<RsConfigKeyValueSet*>(*it) ;
if(vitem != NULL)
for(std::list<RsTlvKeyValue>::const_iterator kit = vitem->tlvkvs.pairs.begin(); kit != vitem->tlvkvs.pairs.end(); ++kit)
{
if(kit->key == "TURTLE_CONFIG_MAX_TR_RATE")
{
int val ;
if (sscanf(kit->value.c_str(), "%d", &val) == 1)
{
setMaxTRForwardRate(val) ;
std::cerr << "Setting max TR forward rate to " << val << std::endl ;
}
}
if(kit->key == "TURTLE_ENABLED")
{
_turtle_routing_enabled = (kit->value == "TRUE") ;
if(!_turtle_routing_enabled)
std::cerr << "WARNING: turtle routing has been disabled. You can enable it again in config->server->turtle router." << std::endl;
}
}
delete *it ;
}
load.clear() ;
return true ;
}
int p3turtle::getMaxTRForwardRate() const
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
return _max_tr_up_rate ;
}
void p3turtle::setMaxTRForwardRate(int val)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
if(val > MAX_TR_FORWARD_PER_SEC_UPPER_LIMIT || val < MAX_TR_FORWARD_PER_SEC_LOWER_LIMIT)
std::cerr << "Warning: MAX_TR_FORWARD_PER_SEC value " << val << " read in config file is off limits [" << MAX_TR_FORWARD_PER_SEC_LOWER_LIMIT << "..." << MAX_TR_FORWARD_PER_SEC_UPPER_LIMIT << "]. Ignoring!" << std::endl;
else
{
_max_tr_up_rate = val ;
std::cerr << "p3turtle: Set max tr up rate to " << val << std::endl;
}
IndicateConfigChanged() ;
}
void p3turtle::getMaxTRForwardRateLimits(int& low,int& high) const
{
low = MAX_TR_FORWARD_PER_SEC_LOWER_LIMIT;
high = MAX_TR_FORWARD_PER_SEC_UPPER_LIMIT;
}
// -----------------------------------------------------------------------------------//
// -------------------------------- Helper functions ------------------------------ //
// -----------------------------------------------------------------------------------//
//
uint32_t p3turtle::generateRandomRequestId()
{
return RSRandom::random_u32() ;
}
uint32_t p3turtle::generatePersonalFilePrint(const TurtleFileHash& hash,uint32_t seed,bool b)
{
// whatever cooking from the file hash and OwnId that cannot be recovered.
// The only important thing is that the saem couple (hash,SSL id) produces the same tunnel
// id. The result uses a boolean to allow generating non symmetric tunnel ids.
std::string buff(hash.toStdString() + _own_id.toStdString()) ;
uint32_t res = seed ;
uint32_t decal = 0 ;
for(int i=0;i<(int)buff.length();++i)
{
res += 7*buff[i] + decal ;
if(b)
decal = decal*44497+15641+(res%86243) ;
else
decal = decal*86243+15649+(res%44497) ;
}
return res ;
}
// -----------------------------------------------------------------------------------//
// -------------------------------- Global routing. -------------------------------- //
// -----------------------------------------------------------------------------------//
//
int p3turtle::handleIncoming()
{
int nhandled = 0;
// While messages read
//
RsItem *item = NULL;
while(NULL != (item = recvItem()))
{
nhandled++;
if(!(_turtle_routing_enabled && _turtle_routing_session_enabled))
delete item ;
else
{
RsTurtleGenericTunnelItem *gti = dynamic_cast<RsTurtleGenericTunnelItem *>(item) ;
if(gti != NULL)
routeGenericTunnelItem(gti) ; /// Generic packets, that travel through established tunnels.
else /// These packets should be destroyed by the client.
{
/// Special packets that require specific treatment, because tunnels do not exist for these packets.
/// These packets are destroyed here, after treatment.
//
switch(item->PacketSubType())
{
case RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST:
case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST:
case RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST: handleSearchRequest(dynamic_cast<RsTurtleSearchRequestItem *>(item)) ;
break ;
case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT :
case RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT : handleSearchResult(dynamic_cast<RsTurtleSearchResultItem *>(item)) ;
break ;
case RS_TURTLE_SUBTYPE_OPEN_TUNNEL : handleTunnelRequest(dynamic_cast<RsTurtleOpenTunnelItem *>(item)) ;
break ;
case RS_TURTLE_SUBTYPE_TUNNEL_OK : handleTunnelResult(dynamic_cast<RsTurtleTunnelOkItem *>(item)) ;
break ;
default:
std::cerr << "p3turtle::handleIncoming: Unknown packet subtype " << item->PacketSubType() << std::endl ;
}
delete item;
}
}
}
return nhandled;
}
// -----------------------------------------------------------------------------------//
// -------------------------------- Search handling. ------------------------------- //
// -----------------------------------------------------------------------------------//
//
void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
{
Dbg3() << __PRETTY_FUNCTION__ << " " << *item << std::endl;
// take a look at the item and test against inconsistent values
// - If the item destimation is
#ifdef P3TURTLE_DEBUG
std::cerr << "Received search request from peer " << item->PeerId() << ": " << std::endl ;
item->print(std::cerr,0) ;
#endif
uint32_t item_size = RsTurtleSerialiser().size(item);
if(item_size > TURTLE_MAX_SEARCH_REQ_ACCEPTED_SERIAL_SIZE)
{
RsWarn() << __PRETTY_FUNCTION__
<< " Got a turtle search item with arbitrary large size from "
<< item->PeerId() << " of size " << item_size << " and depth "
<< item->depth << ". This is not allowed => dropping."
<< std::endl;
return;
}
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
if(_search_requests_origins.size() > MAX_ALLOWED_SR_IN_CACHE)
{
RsWarn() << __PRETTY_FUNCTION__ << " More than "
<< MAX_ALLOWED_SR_IN_CACHE << " search request in cache. "
<< "A peer is probably trying to flood your network See "
"the depth charts to find him." << std::endl;
return;
}
if( _search_requests_origins.find(item->request_id) !=
_search_requests_origins.end() )
{
/* If the item contains an already handled search request, give up.
* This happens when the same search request gets relayed by
* different peers */
return;
}
}
// Perform local search off-mutex,because this might call some services that are above turtle in the mutex chain.
uint32_t search_result_count = 0;
uint32_t max_allowed_hits = TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT;
if(item->PeerId() != _own_id) // is the request not coming from us?
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Request not from us. Performing local search" << std::endl ;
#endif
std::list<RsTurtleSearchResultItem*> search_results ;
performLocalSearch(item,search_result_count,search_results,max_allowed_hits) ;
for(auto it(search_results.begin());it!=search_results.end();++it)
{
(*it)->request_id = item->request_id ;
(*it)->PeerId(item->PeerId()) ;
#ifdef P3TURTLE_DEBUG
std::cerr << " sending back search result for request " << item->request_id << " to back to peer " << item->PeerId() << std::endl ;
#endif
sendItem(*it) ;
}
}
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
// This is a new request. Let's add it to the request map, and forward it to
// open peers.
TurtleSearchRequestInfo& req( _search_requests_origins[item->request_id] ) ;
req.origin = item->PeerId() ;
req.time_stamp = time(NULL) ;
req.depth = item->depth ;
req.result_count = search_result_count;
req.keywords = item->GetKeywords() ;
req.service_id = item->serviceId() ;
req.max_allowed_hits = max_allowed_hits;
// if enough has been sent back already, do not sarch further
#ifdef P3TURTLE_DEBUG
std::cerr << " result count = " << req.result_count << std::endl;
#endif
if(req.result_count >= max_allowed_hits)
return ;
// If search depth not too large, also forward this search request to all other peers.
//
// We use a random factor on the depth test that is biased by a mix between the session id and the partial tunnel id
// to scramble a possible search-by-depth attack.
//
bool random_bypass = (item->depth >= TURTLE_MAX_SEARCH_DEPTH && (((_random_bias ^ item->request_id)&0x7)==2)) ;
bool random_dshift = (item->depth == 1 && (((_random_bias ^ item->request_id)&0x7)==6)) ;
if(item->depth < TURTLE_MAX_SEARCH_DEPTH || random_bypass)
{
std::set<RsPeerId> onlineIds ;
mServiceControl->getPeersConnected(_service_type, onlineIds);
#ifdef P3TURTLE_DEBUG
std::cerr << " Looking for online peers" << std::endl ;
#endif
for(std::set<RsPeerId>::const_iterator it(onlineIds.begin());it!=onlineIds.end();++it)
{
// if(!mServiceControl->isPeerConnected(RS_SERVICE_TYPE_TURTLE,*it))
// continue ;
uint32_t linkType = mLinkMgr->getLinkType(*it);
if ((linkType & RS_NET_CONN_SPEED_TRICKLE) || (linkType & RS_NET_CONN_SPEED_LOW)) // don't forward searches to slow link types (e.g relay peers)!
continue ;
if(*it != item->PeerId())
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Forwarding request to peer = " << *it << std::endl ;
#endif
// Copy current item and modify it.
RsTurtleSearchRequestItem *fwd_item = item->clone() ;