Check-in [f48d583330]
Logged in as anonymous
Overview
Comment:0.2.3: number of fixes for URL editing, plus now edited url is considered for duplicate removal
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: f48d583330f6af00422e81645f4cacf1baec652342bcdc310620885a906f472a
User & Date: arcade on 2021-11-20 08:52:16.222
Other Links: manifest | tags
Context
2021-12-08
13:14
0.2.4: sources are enabled by default when adding, minimize type conversions check-in: 659724c658 user: arcade tags: trunk
2021-11-20
08:52
0.2.3: number of fixes for URL editing, plus now edited url is considered for duplicate removal check-in: f48d583330 user: arcade tags: trunk
2021-11-15
20:55
number of fixes and improvements check-in: 613a665847 user: arcade tags: trunk
Changes
Side-by-Side Diff Ignore Whitespace Patch
Modified Cargo.lock from [9911509215] to [2526269252].
︙
50
51
52
53
54
55
56
57

58
59

60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

79
80
81
82
83
84
85

50
51
52
53
54
55
56

57
58

59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

78
79
80
81
82
83
84
85








-
+

-
+


















-
+








checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2"
dependencies = [
 "alloc-no-stdlib",
]

[[package]]
name = "anyhow"
version = "1.0.45"
version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee10e43ae4a853c0a3591d4e2ada1719e553be18199d9da9d4a83f5927c2f5c7"
checksum = "38d9ff5d688f1c13395289f67db01d4826b46dd694e7580accdc3e8430f2d98e"

[[package]]
name = "arrayvec"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"

[[package]]
name = "async-compression"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443ccbb270374a2b1055fc72da40e1f237809cd6bb0e97e66d264cd138473a6"
dependencies = [
 "brotli",
 "flate2",
 "futures-core",
 "memchr",
 "pin-project-lite",
 "tokio 1.13.0",
 "tokio 1.14.0",
]

[[package]]
name = "atoi"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "616896e05fc0e2649463a93a15183c6a16bf03413a7af88ef1285ddedfa9cda5"
︙
895
896
897
898
899
900
901
902

903
904
905
906
907
908
909

895
896
897
898
899
900
901

902
903
904
905
906
907
908
909








-
+








 "fnv",
 "futures-core",
 "futures-sink",
 "futures-util",
 "http 0.2.5",
 "indexmap",
 "slab",
 "tokio 1.13.0",
 "tokio 1.14.0",
 "tokio-util",
 "tracing",
]

[[package]]
name = "hashbrown"
version = "0.11.2"
︙
1005
1006
1007
1008
1009
1010
1011
1012

1013
1014

1015
1016
1017
1018
1019
1020
1021

1005
1006
1007
1008
1009
1010
1011

1012
1013

1014
1015
1016
1017
1018
1019
1020
1021








-
+

-
+








name = "httparse"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503"

[[package]]
name = "httpdate"
version = "1.0.1"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440"
checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"

[[package]]
name = "hyper"
version = "0.12.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c843caf6296fc1f93444735205af9ed4e109a539005abb2564ae1d6fad34c52"
dependencies = [
︙
1041
1042
1043
1044
1045
1046
1047
1048

1049
1050

1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064

1065
1066
1067
1068
1069
1070
1071

1041
1042
1043
1044
1045
1046
1047

1048
1049

1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063

1064
1065
1066
1067
1068
1069
1070
1071








-
+

-
+













-
+








 "tokio-threadpool",
 "tokio-timer",
 "want 0.2.0",
]

[[package]]
name = "hyper"
version = "0.14.14"
version = "0.14.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b91bb1f221b6ea1f1e4371216b70f40748774c2fb5971b450c07773fb92d26b"
checksum = "436ec0091e4f20e655156a30a0df3770fe2900aa301e548e08446ec794b6953c"
dependencies = [
 "bytes 1.1.0",
 "futures-channel",
 "futures-core",
 "futures-util",
 "h2 0.3.7",
 "http 0.2.5",
 "http-body 0.4.4",
 "httparse",
 "httpdate",
 "itoa",
 "pin-project-lite",
 "socket2",
 "tokio 1.13.0",
 "tokio 1.14.0",
 "tower-service",
 "tracing",
 "want 0.3.0",
]

[[package]]
name = "hyper-tls"
︙
1083
1084
1085
1086
1087
1088
1089
1090

1091
1092

1093
1094
1095
1096
1097
1098
1099

1083
1084
1085
1086
1087
1088
1089

1090
1091

1092
1093
1094
1095
1096
1097
1098
1099








-
+

-
+








[[package]]
name = "hyper-tls"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
dependencies = [
 "bytes 1.1.0",
 "hyper 0.14.14",
 "hyper 0.14.15",
 "native-tls",
 "tokio 1.13.0",
 "tokio 1.14.0",
 "tokio-native-tls",
]

[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
︙
1526
1527
1528
1529
1530
1531
1532
1533

1534
1535

1536
1537
1538
1539
1540
1541
1542

1526
1527
1528
1529
1530
1531
1532

1533
1534

1535
1536
1537
1538
1539
1540
1541
1542








-
+

-
+








name = "openssl-probe"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a"

[[package]]
name = "openssl-sys"
version = "0.9.70"
version = "0.9.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6517987b3f8226b5da3661dad65ff7f300cc59fb5ea8333ca191fc65fde3edf"
checksum = "7df13d165e607909b363a4757a6f133f8a818a74e9d3a98d09c6128e15fa4c73"
dependencies = [
 "autocfg 1.0.1",
 "cc",
 "libc",
 "pkg-config",
 "vcpkg",
]
︙
2008
2009
2010
2011
2012
2013
2014
2015

2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028

2029
2030
2031
2032
2033
2034
2035

2008
2009
2010
2011
2012
2013
2014

2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027

2028
2029
2030
2031
2032
2033
2034
2035








-
+












-
+








 "base64 0.13.0",
 "bytes 1.1.0",
 "encoding_rs",
 "futures-core",
 "futures-util",
 "http 0.2.5",
 "http-body 0.4.4",
 "hyper 0.14.14",
 "hyper 0.14.15",
 "hyper-tls 0.5.0",
 "ipnet",
 "js-sys",
 "lazy_static",
 "log",
 "mime",
 "native-tls",
 "percent-encoding 2.1.0",
 "pin-project-lite",
 "serde 1.0.130",
 "serde_json",
 "serde_urlencoded 0.7.0",
 "tokio 1.13.0",
 "tokio 1.14.0",
 "tokio-native-tls",
 "tokio-util",
 "url 2.2.2",
 "wasm-bindgen",
 "wasm-bindgen-futures",
 "web-sys",
 "winreg 0.7.0",
︙
2044
2045
2046
2047
2048
2049
2050
2051

2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066

2067
2068
2069
2070
2071
2072
2073

2044
2045
2046
2047
2048
2049
2050

2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065

2066
2067
2068
2069
2070
2071
2072
2073








-
+














-
+








 "derive_builder 0.9.0",
 "quick-xml 0.17.2",
 "reqwest 0.9.24",
]

[[package]]
name = "rsstg"
version = "0.2.2"
version = "0.2.3"
dependencies = [
 "anyhow",
 "atom_syndication",
 "chrono",
 "config",
 "futures 0.3.17",
 "futures-util",
 "lazy_static",
 "regex",
 "reqwest 0.11.6",
 "rss",
 "sedregex",
 "sqlx",
 "telegram-bot",
 "tokio 1.13.0",
 "tokio 1.14.0",
]

[[package]]
name = "rust-ini"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2"
︙
2202
2203
2204
2205
2206
2207
2208
2209

2210
2211

2212
2213
2214
2215
2216
2217
2218

2202
2203
2204
2205
2206
2207
2208

2209
2210

2211
2212
2213
2214
2215
2216
2217
2218








-
+

-
+








 "proc-macro2",
 "quote",
 "syn",
]

[[package]]
name = "serde_json"
version = "1.0.70"
version = "1.0.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e277c495ac6cd1a01a58d0a0c574568b4d1ddf14f59965c6a58b8d96400b54f3"
checksum = "063bf466a64011ac24040a49009724ee60a57da1b437617ceb32e53ad61bfb19"
dependencies = [
 "itoa",
 "ryu",
 "serde 1.0.130",
]

[[package]]
︙
2390
2391
2392
2393
2394
2395
2396
2397

2398
2399
2400
2401
2402
2403
2404

2390
2391
2392
2393
2394
2395
2396

2397
2398
2399
2400
2401
2402
2403
2404








-
+








name = "sqlx-rt"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d1bd069de53442e7a320f525a6d4deb8bb0621ac7a55f7eccbc2b58b57f43d0"
dependencies = [
 "native-tls",
 "once_cell",
 "tokio 1.13.0",
 "tokio 1.14.0",
 "tokio-native-tls",
]

[[package]]
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
︙
2467
2468
2469
2470
2471
2472
2473
2474

2475
2476
2477
2478

2479
2480
2481
2482
2483
2484
2485

2467
2468
2469
2470
2471
2472
2473

2474
2475
2476
2477

2478
2479
2480
2481
2482
2483
2484
2485








-
+



-
+








[[package]]
name = "telegram-bot"
version = "0.9.0"
source = "git+https://github.com/telegram-rs/telegram-bot#65ad5cfd578e9a1260ce6daac714eb2153c0bec7"
dependencies = [
 "bytes 1.1.0",
 "futures 0.3.17",
 "hyper 0.14.14",
 "hyper 0.14.15",
 "hyper-tls 0.5.0",
 "multipart",
 "telegram-bot-raw",
 "tokio 1.13.0",
 "tokio 1.14.0",
 "tracing",
 "tracing-futures",
]

[[package]]
name = "telegram-bot-raw"
version = "0.9.0"
︙
2568
2569
2570
2571
2572
2573
2574
2575

2576
2577

2578
2579
2580
2581
2582
2583
2584

2568
2569
2570
2571
2572
2573
2574

2575
2576

2577
2578
2579
2580
2581
2582
2583
2584








-
+

-
+








 "tokio-tcp",
 "tokio-threadpool",
 "tokio-timer",
]

[[package]]
name = "tokio"
version = "1.13.0"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "588b2d10a336da58d877567cd8fb8a14b463e2104910f8132cd054b4b96e29ee"
checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144"
dependencies = [
 "autocfg 1.0.1",
 "bytes 1.1.0",
 "libc",
 "memchr",
 "mio 0.7.14",
 "num_cpus",
︙
2627
2628
2629
2630
2631
2632
2633
2634

2635
2636

2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650

2651
2652
2653
2654
2655
2656
2657

2627
2628
2629
2630
2631
2632
2633

2634
2635

2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649

2650
2651
2652
2653
2654
2655
2656
2657








-
+

-
+













-
+








 "bytes 0.4.12",
 "futures 0.1.31",
 "log",
]

[[package]]
name = "tokio-macros"
version = "1.5.1"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "114383b041aa6212c579467afa0075fbbdd0718de036100bc0ba7961d8cb9095"
checksum = "c9efc1aba077437943f7515666aa2b882dfabfbfdf89c819ea75a8d6e9eaba5e"
dependencies = [
 "proc-macro2",
 "quote",
 "syn",
]

[[package]]
name = "tokio-native-tls"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
dependencies = [
 "native-tls",
 "tokio 1.13.0",
 "tokio 1.14.0",
]

[[package]]
name = "tokio-reactor"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351"
︙
2673
2674
2675
2676
2677
2678
2679
2680

2681
2682
2683
2684
2685
2686
2687

2673
2674
2675
2676
2677
2678
2679

2680
2681
2682
2683
2684
2685
2686
2687








-
+








name = "tokio-stream"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3"
dependencies = [
 "futures-core",
 "pin-project-lite",
 "tokio 1.13.0",
 "tokio 1.14.0",
]

[[package]]
name = "tokio-sync"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee"
︙
2740
2741
2742
2743
2744
2745
2746
2747

2748
2749
2750
2751
2752
2753
2754

2740
2741
2742
2743
2744
2745
2746

2747
2748
2749
2750
2751
2752
2753
2754








-
+








checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0"
dependencies = [
 "bytes 1.1.0",
 "futures-core",
 "futures-sink",
 "log",
 "pin-project-lite",
 "tokio 1.13.0",
 "tokio 1.14.0",
]

[[package]]
name = "toml"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa"
︙
Modified Cargo.toml from [87c378d1a8] to [4c44f7ccc3]. 
1
2
3

4
5
6
7
8
9
10

1
2

3
4
5
6
7
8
9
10



-
+








[package]
name = "rsstg"
version = "0.2.2"
version = "0.2.3"
authors = ["arcade"]
edition = "2018"

[dependencies]
anyhow = "*"
atom_syndication = { version = "*", features = [ "with-serde" ] }
chrono = "*"
︙
Modified src/core.rs from [f054b2025f] to [b0c07df6ab].
︙
148
149
150
151
152
153
154




155
156

157
158
159
160
161
162
163
164
165
166
167
168
169
170

171
172
173
174
175
176
177

178
179
180
181
182
183
184

148
149
150
151
152
153
154
155
156
157
158
159

160
161
162
163
164
165
166
167
168




169

170
171
172
173
174
175
176

177
178
179
180
181
182
183
184








+
+
+
+

-
+








-
-
-
-

-
+






-
+








					rss::Error::Eof => (),
					_ => bail!("Unsupported or mangled content:\n{:?}\n{:#?}\n{:#?}\n", &url, err, status)
				}
			};
			for (date, url) in posts.iter() {
				let mut conn = self.pool.acquire().await
					.with_context(|| format!("Check post fetch conn:\n{:?}", &self.pool))?;
				let post_url = match url_re {
					Some(ref x) => x.execute(url).to_string(),
					None => url.to_string(),
				};
				let row = sqlx::query("select exists(select true from rsstg_post where url = $1 and source_id = $2) as exists;")
					.bind(url)
					.bind(&post_url)
					.bind(*id)
					.fetch_one(&mut conn).await
					.with_context(|| format!("Check post:\n{:?}", &conn))?;
				let exists: bool = row.try_get("exists")?;
				if ! exists {
					if this_fetch == None || *date > this_fetch.unwrap() {
						this_fetch = Some(*date);
					};
					let post_url = match url_re {
						Some(ref x) => x.execute(url).to_string(),
						None => url.to_string(),
					};
					self.tg.send( match iv_hash {
							Some(hash) => telegram_bot::SendMessage::new(destination, format!("<a href=\"https://t.me/iv?url={}&rhash={}\"> </a>{0}", post_url, hash)),
							Some(hash) => telegram_bot::SendMessage::new(destination, format!("<a href=\"https://t.me/iv?url={}&rhash={}\"> </a>{0}", &post_url, hash)),
							None => telegram_bot::SendMessage::new(destination, format!("{}", post_url)),
						}.parse_mode(telegram_bot::types::ParseMode::Html)).await
						.context("Can't post message:")?;
					sqlx::query("insert into rsstg_post (source_id, posted, url) values ($1, $2, $3);")
						.bind(*id)
						.bind(date)
						.bind(url)
						.bind(post_url)
						.execute(&mut conn).await
						.with_context(|| format!("Record post:\n{:?}", &conn))?;
					drop(conn);
					tokio::time::sleep(std::time::Duration::new(4, 0)).await;
				};
				posted += 1;
			};
︙