1
// Copyright (C) Moondance Labs Ltd.
2
// This file is part of Tanssi.
3

            
4
// Tanssi is free software: you can redistribute it and/or modify
5
// it under the terms of the GNU General Public License as published by
6
// the Free Software Foundation, either version 3 of the License, or
7
// (at your option) any later version.
8

            
9
// Tanssi is distributed in the hope that it will be useful,
10
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
// GNU General Public License for more details.
13

            
14
// You should have received a copy of the GNU General Public License
15
// along with Tanssi.  If not, see <http://www.gnu.org/licenses/>.
16

            
17
use {
18
    cumulus_client_cli::CollatorOptions,
19
    futures::FutureExt,
20
    log::{info, warn},
21
    node_common::{
22
        cli::RelayChainCli, service::solochain::RelayAsOrchestratorChainInterfaceBuilder,
23
    },
24
    sc_cli::{CliConfiguration, DefaultConfigurationValues, LoggerBuilder, Signals, SubstrateCli},
25
    sc_network::config::NetworkBackendType,
26
    sc_service::{
27
        config::{ExecutorConfiguration, KeystoreConfig, NetworkConfiguration, TransportConfig},
28
        BasePath, BlocksPruning, ChainType, Configuration, DatabaseSource, GenericChainSpec,
29
        KeystoreContainer, NoExtension, Role, TaskManager,
30
    },
31
    sp_keystore::KeystorePtr,
32
    std::{
33
        future::Future,
34
        marker::PhantomData,
35
        num::NonZeroUsize,
36
        path::{Path, PathBuf},
37
        sync::Arc,
38
        time::Duration,
39
    },
40
    tc_consensus::{OrchestratorChainInterface, RelayChainInterface},
41
    tc_service_container_chain_spawner::cli::ContainerChainCli,
42
    tc_service_container_chain_spawner::{
43
        spawner,
44
        spawner::{CcSpawnMsg, ContainerChainSpawnParams, ContainerChainSpawner},
45
    },
46
    tokio::sync::mpsc::unbounded_channel,
47
    tokio_util::sync::CancellationToken,
48
};
49

            
50
#[derive(Copy, Clone, PartialEq, Eq)]
51
pub enum EnableContainerChainSpawner {
52
    Yes,
53
    No,
54
}
55

            
56
pub struct SolochainNodeStarted {
57
    pub task_manager: TaskManager,
58
    pub relay_chain_interface: Arc<dyn RelayChainInterface>,
59
    pub orchestrator_chain_interface: Arc<dyn OrchestratorChainInterface>,
60
    pub keystore: KeystorePtr,
61
}
62

            
63
/// Start a solochain node.
64
pub async fn start_solochain_node(
65
    polkadot_config: Configuration,
66
    container_chain_cli: ContainerChainCli,
67
    collator_options: CollatorOptions,
68
    hwbench: Option<sc_sysinfo::HwBench>,
69
    // In container chain rpc provider mode, it manages its own spawner.
70
    enable_cc_spawner: EnableContainerChainSpawner,
71
) -> sc_service::error::Result<SolochainNodeStarted> {
72
    let tokio_handle = polkadot_config.tokio_handle.clone();
73
    let orchestrator_para_id = Default::default();
74

            
75
    let chain_type = polkadot_config.chain_spec.chain_type().clone();
76
    let relay_chain = polkadot_config.chain_spec.id().to_string();
77

            
78
    // We use the relaychain keystore config for collators
79
    // Ensure that the user did not provide any custom keystore path for collators
80
    if container_chain_cli
81
        .base
82
        .base
83
        .keystore_params
84
        .keystore_path
85
        .is_some()
86
    {
87
        panic!(
88
            "--keystore-path not allowed here, must be set in relaychain args, after the first --"
89
        )
90
    }
91
    let keystore = &polkadot_config.keystore;
92

            
93
    // Instead of putting keystore in
94
    // Collator1000-01/data/chains/simple_container_2000/keystore
95
    // We put it in
96
    // Collator1000-01/relay-data/chains/dancelight_local_testnet/keystore
97
    // And same for "network" folder
98
    // But zombienet will put the keys in the old path, so we need to manually copy it if we
99
    // are running under zombienet
100
    copy_zombienet_keystore(keystore, container_chain_cli.base_path())?;
101

            
102
    let keystore_container = KeystoreContainer::new(keystore)?;
103

            
104
    // No metrics so no prometheus registry
105
    let prometheus_registry = None;
106
    let mut task_manager = TaskManager::new(tokio_handle.clone(), prometheus_registry)?;
107

            
108
    // Each container chain will spawn its own telemetry
109
    let telemetry_worker_handle = None;
110

            
111
    // Dummy parachain config only needed because `build_relay_chain_interface` needs to know if we
112
    // are collators or not
113
    let validator = container_chain_cli.base.collator;
114

            
115
    let mut dummy_parachain_config = dummy_config(
116
        polkadot_config.tokio_handle.clone(),
117
        polkadot_config.base_path.clone(),
118
    );
119
    dummy_parachain_config.role = if validator {
120
        Role::Authority
121
    } else {
122
        Role::Full
123
    };
124
    let (relay_chain_interface, collator_key) =
125
        cumulus_client_service::build_relay_chain_interface(
126
            polkadot_config,
127
            &dummy_parachain_config,
128
            telemetry_worker_handle.clone(),
129
            &mut task_manager,
130
            collator_options.clone(),
131
            hwbench.clone(),
132
        )
133
        .await
134
        .map_err(|e| sc_service::Error::Application(Box::new(e) as Box<_>))?;
135

            
136
    log::info!("start_solochain_node: is validator? {}", validator);
137

            
138
    let overseer_handle = relay_chain_interface
139
        .overseer_handle()
140
        .map_err(|e| sc_service::Error::Application(Box::new(e)))?;
141
    let sync_keystore = keystore_container.keystore();
142
    let collate_on_tanssi: Arc<
143
        dyn Fn() -> (CancellationToken, futures::channel::oneshot::Receiver<()>) + Send + Sync,
144
    > = Arc::new(move || {
145
        // collate_on_tanssi will not be called in solochains because solochains use a different consensus
146
        // mechanism and need validators instead of collators.
147
        // The runtime enforces this because the orchestrator_chain is never assigned any collators.
148
        panic!("Called collate_on_tanssi on solochain collator. This is unsupported and the runtime shouldn't allow this, it is a bug")
149
    });
150

            
151
    let orchestrator_chain_interface_builder = RelayAsOrchestratorChainInterfaceBuilder {
152
        overseer_handle: overseer_handle.clone(),
153
        relay_chain_interface: relay_chain_interface.clone(),
154
    };
155
    let orchestrator_chain_interface = orchestrator_chain_interface_builder.build();
156
    // Channel to send messages to start/stop container chains
157
    let (cc_spawn_tx, cc_spawn_rx) = unbounded_channel();
158

            
159
    if validator {
160
        if enable_cc_spawner == EnableContainerChainSpawner::No {
161
            panic!("cannot be a validator if container chain spawner is disabled");
162
        }
163

            
164
        // Start task which detects para id assignment, and starts/stops container chains.
165
        crate::build_check_assigned_para_id(
166
            orchestrator_chain_interface.clone(),
167
            sync_keystore.clone(),
168
            cc_spawn_tx.clone(),
169
            task_manager.spawn_essential_handle(),
170
        );
171
    }
172

            
173
    // If the orchestrator chain is running as a full-node, we start a full node for the
174
    // container chain immediately, because only collator nodes detect their container chain
175
    // assignment so otherwise it will never start.
176
    if !validator && enable_cc_spawner == EnableContainerChainSpawner::Yes {
177
        if let Some(container_chain_para_id) = container_chain_cli.base.para_id {
178
            // Spawn new container chain node
179
            cc_spawn_tx
180
                .send(CcSpawnMsg::UpdateAssignment {
181
                    current: Some(container_chain_para_id.into()),
182
                    next: Some(container_chain_para_id.into()),
183
                })
184
                .map_err(|e| sc_service::Error::Application(Box::new(e) as Box<_>))?;
185
        }
186
    }
187

            
188
    if enable_cc_spawner == EnableContainerChainSpawner::Yes {
189
        // Start container chain spawner task. This will start and stop container chains on demand.
190
        let spawn_handle = task_manager.spawn_handle();
191
        let relay_chain_interface = relay_chain_interface.clone();
192
        let orchestrator_chain_interface = orchestrator_chain_interface.clone();
193

            
194
        let container_chain_spawner = ContainerChainSpawner {
195
            params: ContainerChainSpawnParams {
196
                orchestrator_chain_interface,
197
                container_chain_cli,
198
                tokio_handle,
199
                chain_type,
200
                relay_chain,
201
                relay_chain_interface,
202
                sync_keystore,
203
                collation_params: if validator {
204
                    Some(spawner::CollationParams {
205
                        // TODO: all these args must be solochain instead of orchestrator
206
                        orchestrator_client: None,
207
                        orchestrator_tx_pool: None,
208
                        orchestrator_para_id,
209
                        collator_key: collator_key
210
                            .expect("there should be a collator key if we're a validator"),
211
                        solochain: true,
212
                    })
213
                } else {
214
                    None
215
                },
216
                spawn_handle,
217
                data_preserver: false,
218
                generate_rpc_builder:
219
                    tc_service_container_chain_spawner::rpc::GenerateSubstrateRpcBuilder::<
220
                        dancebox_runtime::RuntimeApi,
221
                    >::new(),
222
                override_sync_mode: Some(sc_cli::SyncMode::Warp),
223
                phantom: PhantomData,
224
            },
225
            state: Default::default(),
226
            db_folder_cleanup_done: false,
227
            collate_on_tanssi,
228
            collation_cancellation_constructs: None,
229
        };
230
        let state = container_chain_spawner.state.clone();
231

            
232
        task_manager.spawn_essential_handle().spawn(
233
            "container-chain-spawner-rx-loop",
234
            None,
235
            container_chain_spawner.rx_loop(cc_spawn_rx, validator, true),
236
        );
237

            
238
        task_manager.spawn_essential_handle().spawn(
239
            "container-chain-spawner-debug-state",
240
            None,
241
            tc_service_container_chain_spawner::monitor::monitor_task(state),
242
        );
243
    }
244

            
245
    Ok(SolochainNodeStarted {
246
        task_manager,
247
        relay_chain_interface,
248
        orchestrator_chain_interface,
249
        keystore: keystore_container.keystore(),
250
    })
251
}
252

            
253
/// Alternative to [Configuration] struct used in solochain context.
254
pub struct SolochainConfig {
255
    pub tokio_handle: tokio::runtime::Handle,
256
    pub base_path: BasePath,
257
    pub network_node_name: String,
258
    pub role: Role,
259
    pub relay_chain: String,
260
}
261

            
262
/// Alternative to [Runner](sc_cli::Runner) struct used in solochain context.
263
pub struct SolochainRunner {
264
    config: SolochainConfig,
265
    tokio_runtime: tokio::runtime::Runtime,
266
    signals: Signals,
267
}
268

            
269
impl SolochainRunner {
270
    /// Log information about the node itself.
271
    ///
272
    /// # Example:
273
    ///
274
    /// ```text
275
    /// 2020-06-03 16:14:21 Substrate Node
276
    /// 2020-06-03 16:14:21 ✌️  version 2.0.0-rc3-f4940588c-x86_64-linux-gnu
277
    /// 2020-06-03 16:14:21 ❤️  by Parity Technologies <admin@parity.io>, 2017-2020
278
    /// 2020-06-03 16:14:21 📋 Chain specification: Flaming Fir
279
    /// 2020-06-03 16:14:21 🏷  Node name: jolly-rod-7462
280
    /// 2020-06-03 16:14:21 👤 Role: FULL
281
    /// 2020-06-03 16:14:21 💾 Database: RocksDb at /tmp/c/chains/flamingfir7/db
282
    /// 2020-06-03 16:14:21 ⛓  Native runtime: node-251 (substrate-node-1.tx1.au10)
283
    /// ```
284
    fn print_node_infos(&self) {
285
        use chrono::{offset::Local, Datelike};
286
        type C = ContainerChainCli;
287
        info!("{}", C::impl_name());
288
        info!("✌️  version {}", C::impl_version());
289
        info!(
290
            "❤️  by {}, {}-{}",
291
            C::author(),
292
            C::copyright_start_year(),
293
            Local::now().year()
294
        );
295
        // No chain spec
296
        //info!("📋 Chain specification: {}", config.chain_spec.name());
297
        info!("🏷  Node name: {}", self.config.network_node_name);
298
        info!("👤 Role: {}", self.config.role);
299
        info!(
300
            "💾 Database: {} at {}",
301
            // Container chains only support paritydb
302
            "ParityDb",
303
            // Print base path instead of db path because each container will have its own db in a
304
            // different subdirectory.
305
            self.config.base_path.path().display(),
306
        );
307
    }
308

            
309
    /// A helper function that runs a node with tokio and stops if the process receives the signal
310
    /// `SIGTERM` or `SIGINT`.
311
    pub fn run_node_until_exit<F, E>(
312
        self,
313
        initialize: impl FnOnce(SolochainConfig) -> F,
314
    ) -> std::result::Result<(), E>
315
    where
316
        F: Future<Output = std::result::Result<TaskManager, E>>,
317
        E: std::error::Error + Send + Sync + 'static + From<sc_service::Error>,
318
    {
319
        self.print_node_infos();
320

            
321
        let mut task_manager = self.tokio_runtime.block_on(initialize(self.config))?;
322

            
323
        let res = self
324
            .tokio_runtime
325
            .block_on(self.signals.run_until_signal(task_manager.future().fuse()));
326
        // We need to drop the task manager here to inform all tasks that they should shut down.
327
        //
328
        // This is important to be done before we instruct the tokio runtime to shutdown. Otherwise
329
        // the tokio runtime will wait the full 60 seconds for all tasks to stop.
330
        let task_registry = task_manager.into_task_registry();
331

            
332
        // Give all futures 60 seconds to shutdown, before tokio "leaks" them.
333
        let shutdown_timeout = Duration::from_secs(60);
334
        self.tokio_runtime.shutdown_timeout(shutdown_timeout);
335

            
336
        let running_tasks = task_registry.running_tasks();
337

            
338
        if !running_tasks.is_empty() {
339
            log::error!("Detected running(potentially stalled) tasks on shutdown:");
340
            running_tasks.iter().for_each(|(task, count)| {
341
                let instances_desc = if *count > 1 {
342
                    format!("with {} instances ", count)
343
                } else {
344
                    "".to_string()
345
                };
346

            
347
                if task.is_default_group() {
348
                    log::error!(
349
                        "Task \"{}\" was still running {}after waiting {} seconds to finish.",
350
                        task.name,
351
                        instances_desc,
352
                        shutdown_timeout.as_secs(),
353
                    );
354
                } else {
355
                    log::error!(
356
						"Task \"{}\" (Group: {}) was still running {}after waiting {} seconds to finish.",
357
						task.name,
358
						task.group,
359
						instances_desc,
360
						shutdown_timeout.as_secs(),
361
					);
362
                }
363
            });
364
        }
365

            
366
        res.map_err(Into::into)
367
    }
368
}
369

            
370
/// Equivalent to [Cli::create_runner]
371
pub fn create_runner<T: SubstrateCli + CliConfiguration<DVC>, DVC: DefaultConfigurationValues>(
372
    command: &T,
373
) -> sc_cli::Result<SolochainRunner> {
374
    let tokio_runtime = sc_cli::build_runtime()?;
375

            
376
    // `capture` needs to be called in a tokio context.
377
    // Also capture them as early as possible.
378
    let signals = tokio_runtime.block_on(async { Signals::capture() })?;
379

            
380
    init_cmd(command, &T::support_url(), &T::impl_version())?;
381

            
382
    let base_path = command.base_path()?.unwrap();
383
    let network_node_name = command.node_name()?;
384
    let is_dev = command.is_dev()?;
385
    let role = command.role(is_dev)?;
386
    // This relay chain id is only used when the relay chain args have no `--chain` value
387
    // TODO: check if this works with an external relay rpc / light client
388
    let relay_chain_id = "dancelight_local_testnet".to_string();
389

            
390
    let config = SolochainConfig {
391
        tokio_handle: tokio_runtime.handle().clone(),
392
        base_path,
393
        network_node_name,
394
        role,
395
        relay_chain: relay_chain_id,
396
    };
397

            
398
    Ok(SolochainRunner {
399
        config,
400
        tokio_runtime,
401
        signals,
402
    })
403
}
404

            
405
/// The recommended open file descriptor limit to be configured for the process.
406
const RECOMMENDED_OPEN_FILE_DESCRIPTOR_LIMIT: u64 = 10_000;
407

            
408
/// Equivalent to [CliConfiguration::init]
409
fn init_cmd<T: CliConfiguration<DVC>, DVC: DefaultConfigurationValues>(
410
    this: &T,
411
    support_url: &str,
412
    impl_version: &str,
413
) -> sc_cli::Result<()> {
414
    sp_panic_handler::set(support_url, impl_version);
415

            
416
    let mut logger = LoggerBuilder::new(this.log_filters()?);
417
    logger
418
        .with_log_reloading(this.enable_log_reloading()?)
419
        .with_detailed_output(this.detailed_log_output()?);
420

            
421
    if let Some(tracing_targets) = this.tracing_targets()? {
422
        let tracing_receiver = this.tracing_receiver()?;
423
        logger.with_profiling(tracing_receiver, tracing_targets);
424
    }
425

            
426
    if this.disable_log_color()? {
427
        logger.with_colors(false);
428
    }
429

            
430
    logger.init()?;
431

            
432
    match fdlimit::raise_fd_limit() {
433
        Ok(fdlimit::Outcome::LimitRaised { to, .. }) => {
434
            if to < RECOMMENDED_OPEN_FILE_DESCRIPTOR_LIMIT {
435
                warn!(
436
                    "Low open file descriptor limit configured for the process. \
437
                        Current value: {:?}, recommended value: {:?}.",
438
                    to, RECOMMENDED_OPEN_FILE_DESCRIPTOR_LIMIT,
439
                );
440
            }
441
        }
442
        Ok(fdlimit::Outcome::Unsupported) => {
443
            // Unsupported platform (non-Linux)
444
        }
445
        Err(error) => {
446
            warn!(
447
                "Failed to configure file descriptor limit for the process: \
448
                    {}, recommended value: {:?}.",
449
                error, RECOMMENDED_OPEN_FILE_DESCRIPTOR_LIMIT,
450
            );
451
        }
452
    }
453

            
454
    Ok(())
455
}
456

            
457
/// Equivalent to [RelayChainCli::new]
458
pub fn relay_chain_cli_new<'a>(
459
    config: &SolochainConfig,
460
    relay_chain_args: impl Iterator<Item = &'a String>,
461
) -> RelayChainCli {
462
    let base_path = config.base_path.path().join("polkadot");
463

            
464
    RelayChainCli {
465
        base_path,
466
        chain_id: Some(config.relay_chain.clone()),
467
        base: clap::Parser::parse_from(relay_chain_args),
468
        solochain: true,
469
    }
470
}
471

            
472
/// Create a dummy [Configuration] that should only be used as input to polkadot-sdk functions that
473
/// take this struct as input but only use one field of it.
474
/// This is needed because [Configuration] does not implement [Default].
475
pub fn dummy_config(tokio_handle: tokio::runtime::Handle, base_path: BasePath) -> Configuration {
476
    Configuration {
477
        impl_name: "".to_string(),
478
        impl_version: "".to_string(),
479
        role: Role::Full,
480
        tokio_handle,
481
        transaction_pool: Default::default(),
482
        network: NetworkConfiguration {
483
            net_config_path: None,
484
            listen_addresses: vec![],
485
            public_addresses: vec![],
486
            boot_nodes: vec![],
487
            node_key: Default::default(),
488
            default_peers_set: Default::default(),
489
            default_peers_set_num_full: 0,
490
            client_version: "".to_string(),
491
            node_name: "".to_string(),
492
            transport: TransportConfig::MemoryOnly,
493
            max_parallel_downloads: 0,
494
            max_blocks_per_request: 0,
495
            sync_mode: Default::default(),
496
            enable_dht_random_walk: false,
497
            allow_non_globals_in_dht: false,
498
            kademlia_disjoint_query_paths: false,
499
            kademlia_replication_factor: NonZeroUsize::new(20).unwrap(),
500
            ipfs_server: false,
501
            network_backend: Some(NetworkBackendType::Libp2p),
502
        },
503
        keystore: KeystoreConfig::InMemory,
504
        database: DatabaseSource::ParityDb {
505
            path: Default::default(),
506
        },
507
        trie_cache_maximum_size: None,
508
        state_pruning: None,
509
        blocks_pruning: BlocksPruning::KeepAll,
510
        chain_spec: Box::new(
511
            GenericChainSpec::<NoExtension, ()>::builder(Default::default(), NoExtension::None)
512
                .with_name("test")
513
                .with_id("test_id")
514
                .with_chain_type(ChainType::Development)
515
                .with_genesis_config_patch(Default::default())
516
                .build(),
517
        ),
518
        executor: ExecutorConfiguration {
519
            wasm_method: Default::default(),
520
            wasmtime_precompiled: None,
521
            default_heap_pages: None,
522
            max_runtime_instances: 0,
523
            runtime_cache_size: 0,
524
        },
525
        wasm_runtime_overrides: None,
526
        rpc: sc_service::config::RpcConfiguration {
527
            addr: None,
528
            max_connections: 0,
529
            cors: None,
530
            methods: Default::default(),
531
            max_request_size: 0,
532
            max_response_size: 0,
533
            id_provider: None,
534
            max_subs_per_conn: 0,
535
            port: 0,
536
            message_buffer_capacity: 0,
537
            batch_config: jsonrpsee::server::BatchRequestConfig::Disabled,
538
            rate_limit: None,
539
            rate_limit_whitelisted_ips: vec![],
540
            rate_limit_trust_proxy_headers: false,
541
        },
542
        prometheus_config: None,
543
        telemetry_endpoints: None,
544
        offchain_worker: Default::default(),
545
        force_authoring: false,
546
        disable_grandpa: false,
547
        dev_key_seed: None,
548
        tracing_targets: None,
549
        tracing_receiver: Default::default(),
550
        announce_block: false,
551
        data_path: Default::default(),
552
        base_path,
553
    }
554
}
555

            
556
/// Get the zombienet keystore path from the container base path.
557
fn zombienet_keystore_path(container_base_path: &Path) -> PathBuf {
558
    // container base path:
559
    // Collator-01/data/containers
560
    let mut zombienet_path = container_base_path.to_owned();
561
    zombienet_path.pop();
562
    // Collator-01/data/
563
    zombienet_path.push("chains/simple_container_2000/keystore/");
564
    // Collator-01/data/chains/simple_container_2000/keystore/
565

            
566
    zombienet_path
567
}
568

            
569
/// When running under zombienet, collator keys are injected in a different folder from what we
570
/// expect. This function will check if the zombienet folder exists, and if so, copy all the keys
571
/// from there into the expected folder.
572
pub fn copy_zombienet_keystore(
573
    keystore: &KeystoreConfig,
574
    container_base_path: sc_cli::Result<Option<BasePath>>,
575
) -> std::io::Result<()> {
576
    let container_base_path = match container_base_path {
577
        Ok(Some(base_path)) => base_path,
578
        _ => {
579
            // If base_path is not explicitly set, we are not running under zombienet, so there is nothing to do
580
            return Ok(());
581
        }
582
    };
583
    let keystore_path = keystore.path();
584
    let keystore_path = match keystore_path {
585
        Some(x) => x,
586
        None => {
587
            // In-memory keystore, zombienet does not use it by default so ignore it
588
            return Ok(());
589
        }
590
    };
591
    let zombienet_path = zombienet_keystore_path(container_base_path.path());
592

            
593
    if zombienet_path.exists() {
594
        // Copy to keystore folder
595
        let mut files_copied = 0;
596
        copy_dir_all(zombienet_path, keystore_path, &mut files_copied)?;
597
        log::info!("Copied {} keys from zombienet keystore", files_copied);
598

            
599
        Ok(())
600
    } else {
601
        // Zombienet folder does not exist, assume we are not running under zombienet
602
        Ok(())
603
    }
604
}
605

            
606
/// Equivalent to `cp -r src/* dst`
607
// https://stackoverflow.com/a/65192210
608
fn copy_dir_all(
609
    src: impl AsRef<Path>,
610
    dst: impl AsRef<Path>,
611
    files_copied: &mut u32,
612
) -> std::io::Result<()> {
613
    use std::fs;
614
    fs::create_dir_all(&dst)?;
615
    // no-op if src and dst are the same dir
616
    let src_root = src.as_ref().canonicalize()?;
617
    let dst_root = dst.as_ref().canonicalize()?;
618
    if src_root == dst_root {
619
        return Ok(());
620
    }
621
    for entry in fs::read_dir(src)? {
622
        let entry = entry?;
623
        let ty = entry.file_type()?;
624
        if ty.is_dir() {
625
            copy_dir_all(
626
                entry.path(),
627
                dst.as_ref().join(entry.file_name()),
628
                files_copied,
629
            )?;
630
        } else {
631
            fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?;
632
            *files_copied += 1;
633
        }
634
    }
635
    Ok(())
636
}