Overview
Comment: | 0.2.3: number of fixes for URL editing, plus now edited url is considered for duplicate removal |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA3-256: |
f48d583330f6af00422e81645f4cacf1 |
User & Date: | arcade on 2021-11-20 08:52:16.222 |
Other Links: | manifest | tags |
Context
2021-12-08
| ||
13:14 | 0.2.4: sources are enabled by default when adding, minimize type conversions check-in: 659724c658 user: arcade tags: trunk | |
2021-11-20
| ||
08:52 | 0.2.3: number of fixes for URL editing, plus now edited url is considered for duplicate removal check-in: f48d583330 user: arcade tags: trunk | |
2021-11-15
| ||
20:55 | number of fixes and improvements check-in: 613a665847 user: arcade tags: trunk | |
Changes
Modified Cargo.lock
from [9911509215]
to [2526269252].
︙ | ︙ | |||
50 51 52 53 54 55 56 | checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2" dependencies = [ "alloc-no-stdlib", ] [[package]] name = "anyhow" | | | | | 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2" dependencies = [ "alloc-no-stdlib", ] [[package]] name = "anyhow" version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38d9ff5d688f1c13395289f67db01d4826b46dd694e7580accdc3e8430f2d98e" [[package]] name = "arrayvec" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "async-compression" version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443ccbb270374a2b1055fc72da40e1f237809cd6bb0e97e66d264cd138473a6" dependencies = [ "brotli", "flate2", "futures-core", "memchr", "pin-project-lite", "tokio 1.14.0", ] [[package]] name = "atoi" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "616896e05fc0e2649463a93a15183c6a16bf03413a7af88ef1285ddedfa9cda5" |
︙ | ︙ | |||
895 896 897 898 899 900 901 | "fnv", "futures-core", "futures-sink", "futures-util", "http 0.2.5", "indexmap", "slab", | | | 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 | "fnv", "futures-core", "futures-sink", "futures-util", "http 0.2.5", "indexmap", "slab", "tokio 1.14.0", "tokio-util", "tracing", ] [[package]] name = "hashbrown" version = "0.11.2" |
︙ | ︙ | |||
1005 1006 1007 1008 1009 1010 1011 | name = "httparse" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" [[package]] name = "httpdate" | | | | 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 | name = "httparse" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" [[package]] name = "httpdate" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" version = "0.12.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c843caf6296fc1f93444735205af9ed4e109a539005abb2564ae1d6fad34c52" dependencies = [ |
︙ | ︙ | |||
1041 1042 1043 1044 1045 1046 1047 | "tokio-threadpool", "tokio-timer", "want 0.2.0", ] [[package]] name = "hyper" | | | | | 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 | "tokio-threadpool", "tokio-timer", "want 0.2.0", ] [[package]] name = "hyper" version = "0.14.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "436ec0091e4f20e655156a30a0df3770fe2900aa301e548e08446ec794b6953c" dependencies = [ "bytes 1.1.0", "futures-channel", "futures-core", "futures-util", "h2 0.3.7", "http 0.2.5", "http-body 0.4.4", "httparse", "httpdate", "itoa", "pin-project-lite", "socket2", "tokio 1.14.0", "tower-service", "tracing", "want 0.3.0", ] [[package]] name = "hyper-tls" |
︙ | ︙ | |||
1083 1084 1085 1086 1087 1088 1089 | [[package]] name = "hyper-tls" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes 1.1.0", | | | | 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 | [[package]] name = "hyper-tls" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes 1.1.0", "hyper 0.14.15", "native-tls", "tokio 1.14.0", "tokio-native-tls", ] [[package]] name = "ident_case" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" |
︙ | ︙ | |||
1526 1527 1528 1529 1530 1531 1532 | name = "openssl-probe" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" [[package]] name = "openssl-sys" | | | | 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 | name = "openssl-probe" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" [[package]] name = "openssl-sys" version = "0.9.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df13d165e607909b363a4757a6f133f8a818a74e9d3a98d09c6128e15fa4c73" dependencies = [ "autocfg 1.0.1", "cc", "libc", "pkg-config", "vcpkg", ] |
︙ | ︙ | |||
2008 2009 2010 2011 2012 2013 2014 | "base64 0.13.0", "bytes 1.1.0", "encoding_rs", "futures-core", "futures-util", "http 0.2.5", "http-body 0.4.4", | | | | 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 | "base64 0.13.0", "bytes 1.1.0", "encoding_rs", "futures-core", "futures-util", "http 0.2.5", "http-body 0.4.4", "hyper 0.14.15", "hyper-tls 0.5.0", "ipnet", "js-sys", "lazy_static", "log", "mime", "native-tls", "percent-encoding 2.1.0", "pin-project-lite", "serde 1.0.130", "serde_json", "serde_urlencoded 0.7.0", "tokio 1.14.0", "tokio-native-tls", "tokio-util", "url 2.2.2", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", "winreg 0.7.0", |
︙ | ︙ | |||
2044 2045 2046 2047 2048 2049 2050 | "derive_builder 0.9.0", "quick-xml 0.17.2", "reqwest 0.9.24", ] [[package]] name = "rsstg" | | | | 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 | "derive_builder 0.9.0", "quick-xml 0.17.2", "reqwest 0.9.24", ] [[package]] name = "rsstg" version = "0.2.3" dependencies = [ "anyhow", "atom_syndication", "chrono", "config", "futures 0.3.17", "futures-util", "lazy_static", "regex", "reqwest 0.11.6", "rss", "sedregex", "sqlx", "telegram-bot", "tokio 1.14.0", ] [[package]] name = "rust-ini" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" |
︙ | ︙ | |||
2202 2203 2204 2205 2206 2207 2208 | "proc-macro2", "quote", "syn", ] [[package]] name = "serde_json" | | | | 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 | "proc-macro2", "quote", "syn", ] [[package]] name = "serde_json" version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063bf466a64011ac24040a49009724ee60a57da1b437617ceb32e53ad61bfb19" dependencies = [ "itoa", "ryu", "serde 1.0.130", ] [[package]] |
︙ | ︙ | |||
2390 2391 2392 2393 2394 2395 2396 | name = "sqlx-rt" version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d1bd069de53442e7a320f525a6d4deb8bb0621ac7a55f7eccbc2b58b57f43d0" dependencies = [ "native-tls", "once_cell", | | | 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 | name = "sqlx-rt" version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d1bd069de53442e7a320f525a6d4deb8bb0621ac7a55f7eccbc2b58b57f43d0" dependencies = [ "native-tls", "once_cell", "tokio 1.14.0", "tokio-native-tls", ] [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" |
︙ | ︙ | |||
2467 2468 2469 2470 2471 2472 2473 | [[package]] name = "telegram-bot" version = "0.9.0" source = "git+https://github.com/telegram-rs/telegram-bot#65ad5cfd578e9a1260ce6daac714eb2153c0bec7" dependencies = [ "bytes 1.1.0", "futures 0.3.17", | | | | 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 | [[package]] name = "telegram-bot" version = "0.9.0" source = "git+https://github.com/telegram-rs/telegram-bot#65ad5cfd578e9a1260ce6daac714eb2153c0bec7" dependencies = [ "bytes 1.1.0", "futures 0.3.17", "hyper 0.14.15", "hyper-tls 0.5.0", "multipart", "telegram-bot-raw", "tokio 1.14.0", "tracing", "tracing-futures", ] [[package]] name = "telegram-bot-raw" version = "0.9.0" |
︙ | ︙ | |||
2568 2569 2570 2571 2572 2573 2574 | "tokio-tcp", "tokio-threadpool", "tokio-timer", ] [[package]] name = "tokio" | | | | 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 | "tokio-tcp", "tokio-threadpool", "tokio-timer", ] [[package]] name = "tokio" version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144" dependencies = [ "autocfg 1.0.1", "bytes 1.1.0", "libc", "memchr", "mio 0.7.14", "num_cpus", |
︙ | ︙ | |||
2627 2628 2629 2630 2631 2632 2633 | "bytes 0.4.12", "futures 0.1.31", "log", ] [[package]] name = "tokio-macros" | | | | | 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 | "bytes 0.4.12", "futures 0.1.31", "log", ] [[package]] name = "tokio-macros" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9efc1aba077437943f7515666aa2b882dfabfbfdf89c819ea75a8d6e9eaba5e" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "tokio-native-tls" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", "tokio 1.14.0", ] [[package]] name = "tokio-reactor" version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" |
︙ | ︙ | |||
2673 2674 2675 2676 2677 2678 2679 | name = "tokio-stream" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" dependencies = [ "futures-core", "pin-project-lite", | | | 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 | name = "tokio-stream" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" dependencies = [ "futures-core", "pin-project-lite", "tokio 1.14.0", ] [[package]] name = "tokio-sync" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" |
︙ | ︙ | |||
2740 2741 2742 2743 2744 2745 2746 | checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" dependencies = [ "bytes 1.1.0", "futures-core", "futures-sink", "log", "pin-project-lite", | | | 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 | checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" dependencies = [ "bytes 1.1.0", "futures-core", "futures-sink", "log", "pin-project-lite", "tokio 1.14.0", ] [[package]] name = "toml" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" |
︙ | ︙ |
Modified Cargo.toml
from [87c378d1a8]
to [4c44f7ccc3].
1 2 | [package] name = "rsstg" | | | 1 2 3 4 5 6 7 8 9 10 | [package] name = "rsstg" version = "0.2.3" authors = ["arcade"] edition = "2018" [dependencies] anyhow = "*" atom_syndication = { version = "*", features = [ "with-serde" ] } chrono = "*" |
︙ | ︙ |
Modified src/core.rs
from [f054b2025f]
to [b0c07df6ab].
︙ | ︙ | |||
148 149 150 151 152 153 154 155 | rss::Error::Eof => (), _ => bail!("Unsupported or mangled content:\n{:?}\n{:#?}\n{:#?}\n", &url, err, status) } }; for (date, url) in posts.iter() { let mut conn = self.pool.acquire().await .with_context(|| format!("Check post fetch conn:\n{:?}", &self.pool))?; let row = sqlx::query("select exists(select true from rsstg_post where url = $1 and source_id = $2) as exists;") | > > > > | < < < < | | | 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 | rss::Error::Eof => (), _ => bail!("Unsupported or mangled content:\n{:?}\n{:#?}\n{:#?}\n", &url, err, status) } }; for (date, url) in posts.iter() { let mut conn = self.pool.acquire().await .with_context(|| format!("Check post fetch conn:\n{:?}", &self.pool))?; let post_url = match url_re { Some(ref x) => x.execute(url).to_string(), None => url.to_string(), }; let row = sqlx::query("select exists(select true from rsstg_post where url = $1 and source_id = $2) as exists;") .bind(&post_url) .bind(*id) .fetch_one(&mut conn).await .with_context(|| format!("Check post:\n{:?}", &conn))?; let exists: bool = row.try_get("exists")?; if ! exists { if this_fetch == None || *date > this_fetch.unwrap() { this_fetch = Some(*date); }; self.tg.send( match iv_hash { Some(hash) => telegram_bot::SendMessage::new(destination, format!("<a href=\"https://t.me/iv?url={}&rhash={}\"> </a>{0}", &post_url, hash)), None => telegram_bot::SendMessage::new(destination, format!("{}", post_url)), }.parse_mode(telegram_bot::types::ParseMode::Html)).await .context("Can't post message:")?; sqlx::query("insert into rsstg_post (source_id, posted, url) values ($1, $2, $3);") .bind(*id) .bind(date) .bind(post_url) .execute(&mut conn).await .with_context(|| format!("Record post:\n{:?}", &conn))?; drop(conn); tokio::time::sleep(std::time::Duration::new(4, 0)).await; }; posted += 1; }; |
︙ | ︙ |