Index: Cargo.lock ================================================================== --- Cargo.lock +++ Cargo.lock @@ -98,11 +98,11 @@ "brotli", "flate2", "futures-core", "memchr", "pin-project-lite", - "tokio 1.28.1", + "tokio 1.28.2", ] [[package]] name = "async-executor" version = "1.5.1" @@ -128,11 +128,11 @@ "async-io", "async-lock", "blocking", "futures-lite", "once_cell", - "tokio 1.28.1", + "tokio 1.28.2", ] [[package]] name = "async-io" version = "1.13.0" @@ -219,11 +219,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.17", + "syn 2.0.18", ] [[package]] name = "atoi" version = "1.0.0" @@ -602,20 +602,10 @@ dependencies = [ "generic-array", "typenum", ] -[[package]] -name = "ctor" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" -dependencies = [ - "quote", - "syn 1.0.109", -] - [[package]] name = "darling" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" @@ -1023,11 +1013,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.17", + "syn 2.0.18", ] [[package]] name = "futures-rustls" version = "0.22.2" @@ -1149,11 +1139,11 @@ "futures-sink", "futures-util", "http 0.2.9", "indexmap", "slab", - "tokio 1.28.1", + "tokio 1.28.2", "tokio-util", "tracing", ] [[package]] @@ -1331,11 +1321,11 @@ "httparse", "httpdate", "itoa 1.0.6", "pin-project-lite", "socket2", - "tokio 1.28.1", + "tokio 1.28.2", "tower-service", "tracing", "want 0.3.0", ] @@ -1359,11 +1349,11 @@ checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes 1.4.0", "hyper 0.14.26", "native-tls", - "tokio 1.28.1", + "tokio 1.28.2", "tokio-native-tls", ] [[package]] name = "iana-time-zone" @@ -1557,15 +1547,14 @@ "scopeguard", ] [[package]] name = "log" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" dependencies = [ - "cfg-if 1.0.0", "value-bag", ] [[package]] name = "matches" @@ -1809,11 +1798,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.17", + "syn 2.0.18", ] [[package]] name = "openssl-probe" version = "0.1.5" @@ -1937,11 +1926,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.17", + "syn 2.0.18", ] [[package]] name = "pin-project-lite" version = "0.2.9" @@ -2328,12 +2317,13 @@ "percent-encoding 2.2.0", "pin-project-lite", "serde", "serde_json", "serde_urlencoded 0.7.1", - "tokio 1.28.1", + "tokio 1.28.2", "tokio-native-tls", + "tokio-socks", "tokio-util", "tower-service", "url 2.3.1", "wasm-bindgen", "wasm-bindgen-futures", @@ -2367,11 +2357,11 @@ "reqwest 0.9.24", ] [[package]] name = "rsstg" -version = "0.2.17" +version = "0.2.18" dependencies = [ "anyhow", "async-std", "atom_syndication", "chrono", @@ -2540,11 +2530,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.17", + "syn 2.0.18", ] [[package]] name = "serde_json" version = "1.0.96" @@ -2812,13 +2802,13 @@ "unicode-ident", ] [[package]] name = "syn" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45b6ddbb36c5b969c182aec3c4a0bce7df3fbad4b77114706a49aacc80567388" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] @@ -2844,11 +2834,11 @@ "futures 0.3.28", "hyper 0.14.26", "hyper-tls 0.5.0", "multipart", "telegram-bot-raw", - "tokio 1.28.1", + "tokio 1.28.2", "tracing", "tracing-futures", ] [[package]] @@ -2891,11 +2881,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.17", + "syn 2.0.18", ] [[package]] name = "time" version = "0.1.45" @@ -2941,13 +2931,13 @@ "tokio-timer", ] [[package]] name = "tokio" -version = "1.28.1" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg 1.1.0", "bytes 1.4.0", "libc", "mio 0.8.6", @@ -3004,11 +2994,11 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.28.1", + "tokio 1.28.2", ] [[package]] name = "tokio-reactor" version = "0.1.12" @@ -3025,10 +3015,22 @@ "slab", "tokio-executor", "tokio-io", "tokio-sync", ] + +[[package]] +name = "tokio-socks" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51165dfa029d2a65969413a6cc96f354b86b464498702f174a4efa13608fd8c0" +dependencies = [ + "either", + "futures-util", + "thiserror", + "tokio 1.28.2", +] [[package]] name = "tokio-sync" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3089,11 +3091,11 @@ dependencies = [ "bytes 1.4.0", "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.28.1", + "tokio 1.28.2", "tracing", ] [[package]] name = "toml" @@ -3128,11 +3130,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 2.0.17", + "syn 2.0.18", ] [[package]] name = "tracing-core" version = "0.1.31" @@ -3258,17 +3260,13 @@ "rand 0.6.5", ] [[package]] name = "value-bag" -version = "1.0.0-alpha.9" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" -dependencies = [ - "ctor", - "version_check", -] +checksum = "a4d330786735ea358f3bc09eea4caa098569c1c93f342d9aca0514915022fe7e" [[package]] name = "vcpkg" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3344,11 +3342,11 @@ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.17", + "syn 2.0.18", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" @@ -3378,11 +3376,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.17", + "syn 2.0.18", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] Index: Cargo.toml ================================================================== --- Cargo.toml +++ Cargo.toml @@ -1,8 +1,8 @@ [package] name = "rsstg" -version = "0.2.17" +version = "0.2.18" authors = ["arcade"] edition = "2021" [dependencies] anyhow = "*" @@ -12,11 +12,11 @@ config = { version = "*", default-features = false, features = [ "toml" ] } futures = "*" futures-util = "*" lazy_static = "*" regex = "*" -reqwest = { version = "^0.11", features = [ "brotli", "gzip", "deflate" ]} +reqwest = { version = "^0.11", features = [ "brotli", "socks", "deflate" ]} rss = { version = "*", features = [ "from_url" ] } sedregex = "*" sqlx = { version = "*", features = [ "postgres", "tls", "runtime-async-std-rustls", "chrono" ] } telegram-bot = { git = "https://github.com/telegram-rs/telegram-bot" } Index: rsstg.toml.example ================================================================== --- rsstg.toml.example +++ rsstg.toml.example @@ -1,3 +1,4 @@ pg = "postgres://user:password@hostname/database" api_key = "speak with @BotFather to get one" -owner = Your ID (for debugging) +owner = "Your ID (for debugging)" +proxy = "Link to http or socks5 proxy you want to use. Tor works too" Index: src/core.rs ================================================================== --- src/core.rs +++ src/core.rs @@ -19,18 +19,27 @@ owner_chat: telegram_bot::UserId, pub tg: telegram_bot::Api, pub my: telegram_bot::User, pool: sqlx::Pool, sources: Arc>>>, + http_client: reqwest::Client, } impl Core { pub fn new(settings: config::Config) -> Result> { let owner = settings.get_int("owner")?; let api_key = settings.get_string("api_key")?; let tg = telegram_bot::Api::new(api_key); let tg_cloned = tg.clone(); + + let proxy = settings.get_string("proxy")?; + let mut client = reqwest::Client::builder(); + if !proxy.is_empty() { + let proxy = reqwest::Proxy::all(proxy)?; + client = client.proxy(proxy); + } + let http_client = client.build()?; let core = Arc::new(Core { tg, my: task::block_on(async { tg_cloned.send(telegram_bot::GetMe).await })?, @@ -39,10 +48,11 @@ .max_connections(5) .acquire_timeout(std::time::Duration::new(300, 0)) .idle_timeout(std::time::Duration::new(60, 0)) .connect_lazy(&settings.get_string("pg")?)?, sources: Arc::new(Mutex::new(HashSet::new())), + http_client, }); let clone = core.clone(); task::spawn(async move { loop { let delay = match &clone.autofetch().await { @@ -96,10 +106,11 @@ .bind(*id) .bind(owner) .fetch_one(&mut conn).await .with_context(|| format!("Query source:\n{:?}", &self.pool))?; drop(conn); + let channel_id: i64 = row.try_get("channel_id")?; let url: &str = row.try_get("url")?; let iv_hash: Option<&str> = row.try_get("iv_hash")?; let url_re = match row.try_get("url_re")? { Some(x) => Some(sedregex::ReplaceCommand::new(x)?), @@ -109,11 +120,12 @@ true => telegram_bot::UserId::new(channel_id), false => telegram_bot::UserId::new(row.try_get("owner")?), }; let mut this_fetch: Option> = None; let mut posts: BTreeMap, String> = BTreeMap::new(); - let response = reqwest::get(url).await?; + + let response = self.http_client.get(url).send().await?; let status = response.status(); let content = response.bytes().await?; match rss::Channel::read_from(&content[..]) { Ok(feed) => { for item in feed.items() {