Add parsing tokens from files or environment variables

Cargo.lock

@@ -681,6 +681,7 @@ checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
 dependencies = [
  "futures-channel",
  "futures-core",
+ "futures-executor",
  "futures-io",
  "futures-sink",
  "futures-task",
@@ -703,6 +704,17 @@ version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
 
+[[package]]
+name = "futures-executor"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
 [[package]]
 name = "futures-io"
 version = "0.3.28"
@@ -738,10 +750,13 @@ version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
 dependencies = [
+ "futures-channel",
  "futures-core",
+ "futures-io",
  "futures-macro",
  "futures-sink",
  "futures-task",
+ "memchr",
  "pin-project-lite",
  "pin-utils",
  "slab",
@@ -1282,9 +1297,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.18.0"
+version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
 [[package]]
 name = "opaque-debug"
@@ -1614,6 +1629,7 @@ dependencies = [
  "rustls-native-certs",
  "rustls-pemfile",
  "serde",
+ "serial_test",
  "sha2",
  "snowstorm",
  "socket2 0.4.9",
@@ -1808,6 +1824,15 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "scc"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76ad2bbb0ae5100a07b7a6f2ed7ab5fd0045551a4c507989b7a620046ea3efdc"
+dependencies = [
+ "sdd",
+]
+
 [[package]]
 name = "schannel"
 version = "0.1.22"
@@ -1823,6 +1848,12 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
+[[package]]
+name = "sdd"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b84345e4c9bd703274a082fb80caaa99b7612be48dfaa1dd9266577ec412309d"
+
 [[package]]
 name = "security-framework"
 version = "2.9.2"
@@ -1883,6 +1914,31 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serial_test"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b4b487fe2acf240a021cf57c6b2b4903b1e78ca0ecd862a71b71d2a51fed77d"
+dependencies = [
+ "futures",
+ "log",
+ "once_cell",
+ "parking_lot",
+ "scc",
+ "serial_test_derive",
+]
+
+[[package]]
+name = "serial_test_derive"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82fe9db325bcef1fbcde82e078a5cc4efdf787e96b3b9cf45b50b529f2083d67"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+]
+
 [[package]]
 name = "sha1"
 version = "0.10.6"

Cargo.toml

@@ -136,3 +136,6 @@ vergen = { version = "7.4.2", default-features = false, features = [
     "cargo",
 ] }
 anyhow = "1.0"
+
+[dev-dependencies]
+serial_test = "3.1.1"

README.md

@@ -101,12 +101,25 @@ Before heading to the full configuration specification, it's recommend to skim [
 
 See [Transport](./docs/transport.md) for more details about encryption and the `transport` block.
 
+Tokens can also be set through environment variables. The variable `RATHOLE_{service name in uppercase}_TOKEN` can be set or `RATHOLE_DEFAULT_TOKEN` for all services. 
+Tokens are parsed in the following order for "servicex":
+1. (client/server).services.servicex.token
+2. (client/server).services.servicex.token_file
+3. RATHOLE_SERVICEX_TOKEN
+4. (client/server).default_token
+5. (client/server).default_token_file
+6. RATHOLE_DEFAULT_TOKEN
+
+Tokens should be generated by yourself (not on someone's website or on random.com) using a cryptographic pseudorandom generator. On Linux, use `openssl rand -hex 64 > /path/to/key`. Make sure to do this on a system with high entropy. 
+Most systems will have plenty of entropy. The random network delay between packets, using the computer and typing, access latency from your hdd all can be used to create entropy. Just use your system for anything other than extremely repetitive tasks and don't generate the key right after boot. 
+
 Here is the full configuration specification:
 
 ```toml
 [client]
 remote_addr = "example.com:2333" # Necessary. The address of the server
 default_token = "default_token_if_not_specify" # Optional. The default token of services, if they don't define their own ones
+default_token_file = "/path/to/token" # Optional. This will pull the default token from the path specified
 heartbeat_timeout = 40 # Optional. Set to 0 to disable the application-layer heartbeat test. The value must be greater than `server.heartbeat_interval`. Default: 40 seconds
 retry_interval = 1 # Optional. The interval between retry to connect to the server. Default: 1 second
 
@@ -134,6 +147,7 @@ tls = true # If `true` then it will use settings in `client.transport.tls`
 [client.services.service1] # A service that needs forwarding. The name `service1` can change arbitrarily, as long as identical to the name in the server's configuration
 type = "tcp" # Optional. The protocol that needs forwarding. Possible values: ["tcp", "udp"]. Default: "tcp"
 token = "whatever" # Necessary if `client.default_token` not set
+token_file = "/path/to/token" # Necessary if token, default_token, the env var, and default_token_file are unset. 
 local_addr = "127.0.0.1:1081" # Necessary. The address of the service that needs to be forwarded
 nodelay = true # Optional. Override the `client.transport.nodelay` per service
 retry_interval = 1 # Optional. The interval between retry to connect to the server. Default: inherits the global config
@@ -144,6 +158,7 @@ local_addr = "127.0.0.1:1082"
 [server]
 bind_addr = "0.0.0.0:2333" # Necessary. The address that the server listens for clients. Generally only the port needs to be change.
 default_token = "default_token_if_not_specify" # Optional
+default_token_file = "/path/to/token" # Optional. This will pull the default token from the path specified
 heartbeat_interval = 30 # Optional. The interval between two application-layer heartbeat. Set to 0 to disable sending heartbeat. Default: 30 seconds
 
 [server.transport] # Same as `[client.transport]`
@@ -169,6 +184,7 @@ tls = true # If `true` then it will use settings in `server.transport.tls`
 [server.services.service1] # The service name must be identical to the client side
 type = "tcp" # Optional. Same as the client `[client.services.X.type]
 token = "whatever" # Necessary if `server.default_token` not set
+token_file = "/path/to/token" # Necessary if token, default_token, and default_token_file are unset. 
 bind_addr = "0.0.0.0:8081" # Necessary. The address of the service is exposed at. Generally only the port needs to be change.
 nodelay = true # Optional. Same as the client
 

src/config.rs

@@ -1,6 +1,7 @@
 use anyhow::{anyhow, bail, Context, Result};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
+use std::env;
 use std::fmt::{Debug, Formatter};
 use std::ops::Deref;
 use std::path::Path;
@@ -64,6 +65,7 @@ pub struct ClientServiceConfig {
     pub name: String,
     pub local_addr: String,
     pub token: Option<MaskedString>,
+    pub token_file: Option<String>,
     pub nodelay: Option<bool>,
     pub retry_interval: Option<u64>,
 }
@@ -101,6 +103,7 @@ pub struct ServerServiceConfig {
     pub name: String,
     pub bind_addr: String,
     pub token: Option<MaskedString>,
+    pub token_file: Option<String>,
     pub nodelay: Option<bool>,
 }
 
@@ -201,6 +204,7 @@ fn default_client_retry_interval() -> u64 {
 pub struct ClientConfig {
     pub remote_addr: String,
     pub default_token: Option<MaskedString>,
+    pub default_token_file: Option<String>,
     pub services: HashMap<String, ClientServiceConfig>,
     #[serde(default)]
     pub transport: TransportConfig,
@@ -219,6 +223,7 @@ fn default_heartbeat_interval() -> u64 {
 pub struct ServerConfig {
     pub bind_addr: String,
     pub default_token: Option<MaskedString>,
+    pub default_token_file: Option<String>,
     pub services: HashMap<String, ServerServiceConfig>,
     #[serde(default)]
     pub transport: TransportConfig,
@@ -234,15 +239,15 @@ pub struct Config {
 }
 
 impl Config {
-    fn from_str(s: &str) -> Result<Config> {
+    async fn from_str(s: &str) -> Result<Config> {
         let mut config: Config = toml::from_str(s).with_context(|| "Failed to parse the config")?;
 
         if let Some(server) = config.server.as_mut() {
-            Config::validate_server_config(server)?;
+            Config::validate_server_config(server).await?;
         }
 
         if let Some(client) = config.client.as_mut() {
-            Config::validate_client_config(client)?;
+            Config::validate_client_config(client).await?;
         }
 
         if config.server.is_none() && config.client.is_none() {
@@ -252,15 +257,48 @@ impl Config {
         }
     }
 
-    fn validate_server_config(server: &mut ServerConfig) -> Result<()> {
+    async fn parse_token(
+        name: &str,
+        token: &Option<MaskedString>,
+        token_file: &Option<String>,
+        default_token: &Option<MaskedString>,
+    ) -> Option<MaskedString> {
+        if token.is_some() {
+            return token.clone();
+        }
+
+        if let Some(v) = token_file {
+            return fs::read_to_string(v).await.ok().map(MaskedString);
+        }
+
+        if let Ok(v) = env::var(format!("RATHOLE_{}_TOKEN", name.to_uppercase())) {
+            return Some(MaskedString(v));
+        }
+
+        if default_token.is_some() {
+            return default_token.clone();
+        }
+
+        None
+    }
+
+    async fn validate_server_config(server: &mut ServerConfig) -> Result<()> {
+        let default_token = Self::parse_token(
+            "default",
+            &server.default_token,
+            &server.default_token_file,
+            &None,
+        )
+        .await;
+
         // Validate services
         for (name, s) in &mut server.services {
-            s.name = name.clone();
+            s.name.clone_from(name);
+            s.token =
+                Self::parse_token(name.as_str(), &s.token, &s.token_file, &default_token).await;
+
             if s.token.is_none() {
-                s.token = server.default_token.clone();
+                bail!("The token of service {} is not set", name);
-                if s.token.is_none() {
-                    bail!("The token of service {} is not set", name);
-                }
             }
         }
 
@@ -269,15 +307,23 @@ impl Config {
         Ok(())
     }
 
-    fn validate_client_config(client: &mut ClientConfig) -> Result<()> {
+    async fn validate_client_config(client: &mut ClientConfig) -> Result<()> {
+        let default_token = Self::parse_token(
+            "default",
+            &client.default_token,
+            &client.default_token_file,
+            &None,
+        )
+        .await;
+
         // Validate services
         for (name, s) in &mut client.services {
-            s.name = name.clone();
+            s.name.clone_from(name);
+            s.token =
+                Self::parse_token(name.as_str(), &s.token, &s.token_file, &default_token).await;
+
             if s.token.is_none() {
-                s.token = client.default_token.clone();
+                bail!("The token of service {} is not set", name);
-                if s.token.is_none() {
-                    bail!("The token of service {} is not set", name);
-                }
             }
             if s.retry_interval.is_none() {
                 s.retry_interval = Some(client.retry_interval);
@@ -327,7 +373,7 @@ impl Config {
         let s: String = fs::read_to_string(path)
             .await
             .with_context(|| format!("Failed to read the config {:?}", path))?;
-        Config::from_str(&s).with_context(|| {
+        Config::from_str(&s).await.with_context(|| {
             "Configuration is invalid. Please refer to the configuration specification."
         })
     }
@@ -336,9 +382,12 @@ impl Config {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use std::{fs, path::PathBuf};
+    use std::fs;
+    use std::path::PathBuf;
 
     use anyhow::Result;
+    use serial_test::{parallel, serial};
+    use tokio::runtime::Runtime;
 
     fn list_config_files<T: AsRef<Path>>(root: T) -> Result<Vec<PathBuf>> {
         let mut files = Vec::new();
@@ -361,38 +410,42 @@ mod tests {
             .collect())
     }
 
-    #[test]
+    #[tokio::test]
-    fn test_example_config() -> Result<()> {
+    #[parallel]
+    async fn test_example_config() -> Result<()> {
         let paths = get_all_example_config()?;
         for p in paths {
             let s = fs::read_to_string(p)?;
-            Config::from_str(&s)?;
+            Config::from_str(&s).await?;
         }
         Ok(())
     }
 
-    #[test]
+    #[tokio::test]
-    fn test_valid_config() -> Result<()> {
+    #[parallel]
+    async fn test_valid_config() -> Result<()> {
         let paths = list_config_files("tests/config_test/valid_config")?;
         for p in paths {
             let s = fs::read_to_string(p)?;
-            Config::from_str(&s)?;
+            Config::from_str(&s).await?;
         }
         Ok(())
     }
 
-    #[test]
+    #[tokio::test]
-    fn test_invalid_config() -> Result<()> {
+    #[parallel]
+    async fn test_invalid_config() -> Result<()> {
         let paths = list_config_files("tests/config_test/invalid_config")?;
         for p in paths {
             let s = fs::read_to_string(p)?;
-            assert!(Config::from_str(&s).is_err());
+            assert!(Config::from_str(&s).await.is_err());
         }
         Ok(())
     }
 
-    #[test]
+    #[tokio::test]
-    fn test_validate_server_config() -> Result<()> {
+    #[parallel]
+    async fn test_validate_server_config() -> Result<()> {
         let mut cfg = ServerConfig::default();
 
         cfg.services.insert(
@@ -407,11 +460,11 @@ mod tests {
         );
 
         // Missing the token
-        assert!(Config::validate_server_config(&mut cfg).is_err());
+        assert!(Config::validate_server_config(&mut cfg).await.is_err());
 
         // Use the default token
         cfg.default_token = Some("123".into());
-        assert!(Config::validate_server_config(&mut cfg).is_ok());
+        assert!(Config::validate_server_config(&mut cfg).await.is_ok());
         assert_eq!(
             cfg.services
                 .get("foo1")
@@ -426,7 +479,7 @@ mod tests {
 
         // The default token won't override the service token
         cfg.services.get_mut("foo1").unwrap().token = Some("4".into());
-        assert!(Config::validate_server_config(&mut cfg).is_ok());
+        assert!(Config::validate_server_config(&mut cfg).await.is_ok());
         assert_eq!(
             cfg.services
                 .get("foo1")
@@ -441,8 +494,9 @@ mod tests {
         Ok(())
     }
 
-    #[test]
+    #[tokio::test]
-    fn test_validate_client_config() -> Result<()> {
+    #[parallel]
+    async fn test_validate_client_config() -> Result<()> {
         let mut cfg = ClientConfig::default();
 
         cfg.services.insert(
@@ -457,11 +511,12 @@ mod tests {
         );
 
         // Missing the token
-        assert!(Config::validate_client_config(&mut cfg).is_err());
+        println!("{:?}", env::var("DEFAULT_TOKEN").ok());
+        assert!(Config::validate_client_config(&mut cfg).await.is_err());
 
         // Use the default token
         cfg.default_token = Some("123".into());
-        assert!(Config::validate_client_config(&mut cfg).is_ok());
+        assert!(Config::validate_client_config(&mut cfg).await.is_ok());
         assert_eq!(
             cfg.services
                 .get("foo1")
@@ -476,7 +531,7 @@ mod tests {
 
         // The default token won't override the service token
         cfg.services.get_mut("foo1").unwrap().token = Some("4".into());
-        assert!(Config::validate_client_config(&mut cfg).is_ok());
+        assert!(Config::validate_client_config(&mut cfg).await.is_ok());
         assert_eq!(
             cfg.services
                 .get("foo1")
@@ -490,4 +545,42 @@ mod tests {
         );
         Ok(())
     }
+
+    #[serial(env_default_token)]
+    fn read_from_env_var() {
+        let mut cfg = ClientConfig::default();
+
+        cfg.services.insert(
+            "foo1".into(),
+            ClientServiceConfig {
+                service_type: ServiceType::Tcp,
+                name: "foo1".into(),
+                local_addr: "127.0.0.1:80".into(),
+                token: None,
+                ..Default::default()
+            },
+        );
+
+        env::set_var("RATHOLE_DEFAULT_TOKEN", "test-token");
+
+        // Can't .await with tokio::test while env vars are set. There must be a block surrounding the futures.
+        let rt = Runtime::new().unwrap();
+        rt.block_on(async {
+            Config::validate_client_config(&mut cfg).await.unwrap();
+        });
+        assert_eq!(
+            cfg.services
+                .get("foo1")
+                .as_ref()
+                .unwrap()
+                .token
+                .as_ref()
+                .unwrap()
+                .0
+                .as_str(),
+            "test-token"
+        );
+
+        env::remove_var("RATHOLE_DEFAULT_TOKEN");
+    }
 }

tests/config_test/valid_config/token_file.toml

@@ -0,0 +1,47 @@
+[client]
+remote_addr = "example.com:2333" # Necessary. The address of the server
+default_token_file = "tests/token" # Optional. The file that stores the token. Can be stored as plain text in the config and on each service
+
+[client.transport]
+type = "tcp" # Optional. Possible values: ["tcp", "tls"]. Default: "tcp"
+
+[client.transport.tls] # Necessary if `type` is "tls"
+trusted_root = "ca.pem" # Necessary. The certificate of CA that signed the server's certificate
+hostname = "example.com" # Optional. The hostname that the client uses to validate the certificate. If not set, fallback to `client.remote_addr`
+
+[client.transport.noise] # Noise protocol. See `docs/transport.md` for further explanation
+pattern = "Noise_NK_25519_ChaChaPoly_BLAKE2s" # Optional. Default value as shown
+local_private_key = "key_encoded_in_base64" # Optional
+remote_public_key = "key_encoded_in_base64" # Optional
+
+[client.services.service1] # A service that needs forwarding. The name `service1` can change arbitrarily, as long as identical to the name in the server's configuration
+type = "tcp" # Optional. The protocol that needs forwarding. Possible values: ["tcp", "udp"]. Default: "tcp"
+token = "whatever" # Necessary if `client.default_token` not set
+local_addr = "127.0.0.1:1081" # Necessary. The address of the service that needs to be forwarded
+
+[client.services.service2] # Multiple services can be defined
+local_addr = "127.0.0.1:1082"
+
+[server]
+bind_addr = "0.0.0.0:2333" # Necessary. The address that the server listens for clients. Generally only the port needs to be change. 
+default_token = "default_token_if_not_specify" # Optional
+
+[server.transport]
+type = "tcp" # Same as `[client.transport]`
+
+[server.transport.tls] # Necessary if `type` is "tls"
+pkcs12 = "identify.pfx" # Necessary. pkcs12 file of server's certificate and private key
+pkcs12_password = "password" # Necessary. Password of the pkcs12 file
+
+[server.transport.noise] # Same as `[client.transport.noise]`
+pattern = "Noise_NK_25519_ChaChaPoly_BLAKE2s"
+local_private_key = "key_encoded_in_base64" 
+remote_public_key = "key_encoded_in_base64" 
+
+[server.services.service1] # The service name must be identical to the client side
+type = "tcp" # Optional. Same as the client `[client.services.X.type]
+token = "whatever" # Necesary if `server.default_token` not set
+bind_addr = "0.0.0.0:8081" # Necessary. The address of the service is exposed at. Generally only the port needs to be change. 
+
+[server.services.service2] 
+bind_addr = "0.0.0.1:8082"

tests/token

@@ -0,0 +1 @@
+test-file-token