Check-in [613a665847]
Logged in as anonymous
Overview
Comment:number of fixes and improvements
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 613a66584745e11176d8a79393f33c87fa00a028e6987e21c183e508ab1a68eb
User & Date: arcade on 2021-11-15 20:55:11.639
Other Links: manifest | tags
Context
2021-11-20
08:52
0.2.3: number of fixes for URL editing, plus now edited url is considered for duplicate removal check-in: f48d583330 user: arcade tags: trunk
2021-11-15
20:55
number of fixes and improvements check-in: 613a665847 user: arcade tags: trunk
2021-11-13
23:16
fix list, add "-" for disabled fields, some reordering check-in: c1e27b74ed user: arcade tags: trunk
Changes
61
62
63
64
65
66
67



68
69
70
71
72
73

74
75
76
77
78
79
80
81
82
83



84
85
86
87
88
89
90
		bail!("Usernames should be something like \"@\\[a\\-zA\\-Z]\\[a\\-zA\\-Z0\\-9\\_]+\", aren't they?\nNot {:?}", &channel);
	};
	if ! RE_LINK.is_match(&url) {
		bail!("Link should be a link to atom/rss feed, something like \"https://domain/path\".\nNot {:?}", &url);
	}
	let iv_hash = match iv_hash {
		Some(hash) => {



			if ! RE_IV_HASH.is_match(hash) {
				bail!("IV hash should be 14 hex digits.\nNot {:?}", hash);
			};
			match *hash {
				"-" => None,
				thing => Some(thing),

			}
		},
		None => None,
	};
	let url_re = match url_re {
		Some(re) => {
			let _url_rex = ReplaceCommand::new(re).context("Regexp parsing error:")?;
			match *re {
				"-" => None,
				thing => Some(thing),



			}
		},
		None => None,
	};
	let channel_id = i64::from(core.tg.send(telegram_bot::GetChat::new(telegram_bot::types::ChatRef::ChannelUsername(channel.to_string()))).await?.id());
	let chan_adm = core.tg.send(telegram_bot::GetChatAdministrators::new(telegram_bot::types::ChatRef::ChannelUsername(channel.to_string()))).await
		.context("Sorry, I have no access to that chat.")?;







>
>
>
|
|
|
<
<
|
>






<


|
>
>
>







61
62
63
64
65
66
67
68
69
70
71
72
73


74
75
76
77
78
79
80
81

82
83
84
85
86
87
88
89
90
91
92
93
94
		bail!("Usernames should be something like \"@\\[a\\-zA\\-Z]\\[a\\-zA\\-Z0\\-9\\_]+\", aren't they?\nNot {:?}", &channel);
	};
	if ! RE_LINK.is_match(&url) {
		bail!("Link should be a link to atom/rss feed, something like \"https://domain/path\".\nNot {:?}", &url);
	}
	let iv_hash = match iv_hash {
		Some(hash) => {
			match *hash {
				"-" => None,
				thing => {
					if ! RE_IV_HASH.is_match(thing) {
						bail!("IV hash should be 14 hex digits.\nNot {:?}", thing);
					};


					Some(thing)
				},
			}
		},
		None => None,
	};
	let url_re = match url_re {
		Some(re) => {

			match *re {
				"-" => None,
				thing => {
					let _url_rex = ReplaceCommand::new(thing).context("Regexp parsing error:")?;
					Some(thing)
				}
			}
		},
		None => None,
	};
	let channel_id = i64::from(core.tg.send(telegram_bot::GetChat::new(telegram_bot::types::ChatRef::ChannelUsername(channel.to_string()))).await?.id());
	let chan_adm = core.tg.send(telegram_bot::GetChatAdministrators::new(telegram_bot::types::ChatRef::ChannelUsername(channel.to_string()))).await
		.context("Sorry, I have no access to that chat.")?;
112
113
114
115
116
117
118
119


120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166

167

168
169
170
171
172
173
174
175
			};
			let destination = match real {
				true => telegram_bot::UserId::new(channel_id),
				false => telegram_bot::UserId::new(row.try_get("owner")?),
			};
			let mut this_fetch: Option<DateTime<chrono::FixedOffset>> = None;
			let mut posts: BTreeMap<DateTime<chrono::FixedOffset>, String> = BTreeMap::new();
			let content = reqwest::get(url).await?.bytes().await?;


			match rss::Channel::read_from(&content[..]) {
				Ok(feed) => {
					for item in feed.items() {
						match item.link() {
							Some(link) => {
								let date = match item.pub_date() {
									Some(feed_date) => DateTime::parse_from_rfc2822(feed_date),
									None => DateTime::parse_from_rfc3339(&item.dublin_core_ext().unwrap().dates()[0]),
								}?;
								let url = link;
								posts.insert(date.clone(), url.into());
							},
							None => {}
						}
					};
				},
				Err(err) => match err {
					rss::Error::InvalidStartTag => {
						let feed = atom_syndication::Feed::read_from(&content[..])
							.with_context(|| format!("Problem opening feed url:\n{}", &url))?;
						for item in feed.entries() {
							let date = item.published().unwrap();
							let url = item.links()[0].href();
							posts.insert(date.clone(), url.into());
						};
					},
					rss::Error::Eof => (),
					_ => bail!("Unsupported or mangled content:\n{:?}\n{:#?}\n", &url, err)
				}
			};
			for (date, url) in posts.iter() {
				let mut conn = self.pool.acquire().await
					.with_context(|| format!("Check post fetch conn:\n{:?}", &self.pool))?;
				let row = sqlx::query("select exists(select true from rsstg_post where url = $1 and source_id = $2) as exists;")
					.bind(url)
					.bind(*id)
					.fetch_one(&mut conn).await
					.with_context(|| format!("Check post:\n{:?}", &conn))?;
				let exists: bool = row.try_get("exists")?;
				if ! exists {
					if this_fetch == None || *date > this_fetch.unwrap() {
						this_fetch = Some(*date);
					};
					self.tg.send( match iv_hash {
							Some(hash) => telegram_bot::SendMessage::new(destination, format!("<a href=\"https://t.me/iv?url={}&rhash={}\"> </a>{0}", match url_re {
								Some(ref x) => x.execute(url).to_string(),
								None => url.to_string(),

							}, hash)),

							None => telegram_bot::SendMessage::new(destination, format!("{}", url)),
						}.parse_mode(telegram_bot::types::ParseMode::Html)).await
						.context("Can't post message:")?;
					sqlx::query("insert into rsstg_post (source_id, posted, url) values ($1, $2, $3);")
						.bind(*id)
						.bind(date)
						.bind(url)
						.execute(&mut conn).await







|
>
>



















|







|















|
<
|
|
>
|
>
|







112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165

166
167
168
169
170
171
172
173
174
175
176
177
178
			};
			let destination = match real {
				true => telegram_bot::UserId::new(channel_id),
				false => telegram_bot::UserId::new(row.try_get("owner")?),
			};
			let mut this_fetch: Option<DateTime<chrono::FixedOffset>> = None;
			let mut posts: BTreeMap<DateTime<chrono::FixedOffset>, String> = BTreeMap::new();
			let response = reqwest::get(url).await?;
			let status = response.status();
			let content = response.bytes().await?;
			match rss::Channel::read_from(&content[..]) {
				Ok(feed) => {
					for item in feed.items() {
						match item.link() {
							Some(link) => {
								let date = match item.pub_date() {
									Some(feed_date) => DateTime::parse_from_rfc2822(feed_date),
									None => DateTime::parse_from_rfc3339(&item.dublin_core_ext().unwrap().dates()[0]),
								}?;
								let url = link;
								posts.insert(date.clone(), url.into());
							},
							None => {}
						}
					};
				},
				Err(err) => match err {
					rss::Error::InvalidStartTag => {
						let feed = atom_syndication::Feed::read_from(&content[..])
							.with_context(|| format!("Problem opening feed url:\n{}\n{}", &url, status))?;
						for item in feed.entries() {
							let date = item.published().unwrap();
							let url = item.links()[0].href();
							posts.insert(date.clone(), url.into());
						};
					},
					rss::Error::Eof => (),
					_ => bail!("Unsupported or mangled content:\n{:?}\n{:#?}\n{:#?}\n", &url, err, status)
				}
			};
			for (date, url) in posts.iter() {
				let mut conn = self.pool.acquire().await
					.with_context(|| format!("Check post fetch conn:\n{:?}", &self.pool))?;
				let row = sqlx::query("select exists(select true from rsstg_post where url = $1 and source_id = $2) as exists;")
					.bind(url)
					.bind(*id)
					.fetch_one(&mut conn).await
					.with_context(|| format!("Check post:\n{:?}", &conn))?;
				let exists: bool = row.try_get("exists")?;
				if ! exists {
					if this_fetch == None || *date > this_fetch.unwrap() {
						this_fetch = Some(*date);
					};
					let post_url = match url_re {

						Some(ref x) => x.execute(url).to_string(),
						None => url.to_string(),
					};
					self.tg.send( match iv_hash {
							Some(hash) => telegram_bot::SendMessage::new(destination, format!("<a href=\"https://t.me/iv?url={}&rhash={}\"> </a>{0}", post_url, hash)),
							None => telegram_bot::SendMessage::new(destination, format!("{}", post_url)),
						}.parse_mode(telegram_bot::types::ParseMode::Html)).await
						.context("Can't post message:")?;
					sqlx::query("insert into rsstg_post (source_id, posted, url) values ($1, $2, $3);")
						.bind(*id)
						.bind(date)
						.bind(url)
						.execute(&mut conn).await